From ffa4f55fabc5c7da73b8a981fc38d8d18699ad38 Mon Sep 17 00:00:00 2001 From: noonari Date: Fri, 9 Sep 2022 02:41:13 +0500 Subject: [PATCH 1/3] PR feedbacks --- CHANGELOG.md | 4 +-- tests/ops/fixtures/saas/datadog_fixtures.py | 33 +++++++++++++++++++ .../saas/test_datadog_task.py | 8 +++-- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aac194095..a1dc6e099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,8 +37,9 @@ The types of changes are: * Wrap up the email connector - it sends an email with erasure instructions as part of request execution [#1246](https://github.com/ethyca/fidesops/pull/1246) * Mapping Vault environment variables in docker-compose.yml [#1275](https://github.com/ethyca/fidesops/pull/1275) * Foundations for a new "manual_webhook" connector type [#1267](https://github.com/ethyca/fidesops/pull/1267) +* Adds Fixture to Datadog Tests [1269](https://github.com/ethyca/fidesops/pull/1269) -### Docs +### Docs * Fix analytics opt out environment variable name [#1170](https://github.com/ethyca/fidesops/pull/1170) * Added how to view a subject request history and reprocess a subject request [#1164](https://github.com/ethyca/fidesops/pull/1164) @@ -52,7 +53,6 @@ The types of changes are: * Fix for pytest-asyncio bug [#1260](https://github.com/ethyca/fidesops/pull/1260) * Fix download link in privacy center [#1264](https://github.com/ethyca/fidesops/pull/1264) * Make admin ui work when volumes are mounted [#1266](https://github.com/ethyca/fidesops/pull/1266) -* Fixed typo in enum value [#1280](https://github.com/ethyca/fidesops/pull/1280) ## [1.7.2](https://github.com/ethyca/fidesops/compare/1.7.1...1.7.2) diff --git a/tests/ops/fixtures/saas/datadog_fixtures.py b/tests/ops/fixtures/saas/datadog_fixtures.py index 230e52075..f32ecd43e 100644 --- a/tests/ops/fixtures/saas/datadog_fixtures.py +++ b/tests/ops/fixtures/saas/datadog_fixtures.py @@ -2,6 +2,7 @@ import pydash import pytest +import requests from fideslib.db import session from sqlalchemy.orm import Session @@ -95,3 +96,35 @@ def datadog_dataset_config( ) yield dataset dataset.delete(db=db) + + +@pytest.fixture(scope="session") +def datadog_access_data(datadog_secrets, datadog_identity_email): + url = "https://api.datadoghq.com/api/v2/logs/events?filter[from]=0&filter[query]=noonari@xyz.com&filter[to]=now&page[limit]=1000" + + payload = {} + headers = { + "DD-API-KEY": f'{datadog_secrets.get("api_key")}', + "DD-APPLICATION-KEY": f'{datadog_secrets.get("app_key")}', + } + + response = requests.request("GET", url, headers=headers, data=payload) + + if datadog_identity_email not in response.text: + url = "https://http-intake.logs.datadoghq.com/api/v2/logs" + payload = [ + { + "ddsource": "nginx", + "ddtags": "env:staging,version:5.1", + "hostname": "i-012345678", + "message": f"2019-11-19T14:37:58,995 INFO [process.name][20081{i}] Hello noonari@xyz.com", + "service": "payment", + } + for i in range(25) + ] + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "DD-API-KEY": f'{datadog_secrets.get("api_key")}', + } + requests.request("POST", url, headers=headers, json=payload) diff --git a/tests/ops/integration_tests/saas/test_datadog_task.py b/tests/ops/integration_tests/saas/test_datadog_task.py index f2d82f85c..efa567d2e 100644 --- a/tests/ops/integration_tests/saas/test_datadog_task.py +++ b/tests/ops/integration_tests/saas/test_datadog_task.py @@ -28,6 +28,7 @@ async def test_saas_access_request_task( datadog_connection_config, datadog_dataset_config, datadog_identity_email, + datadog_access_data, ) -> None: """Full access request based on the Datadog SaaS config""" @@ -51,9 +52,10 @@ async def test_saas_access_request_task( {"email": datadog_identity_email}, db, ) + key = f"{dataset_name}:events" assert_rows_match( - v[f"{dataset_name}:events"], + v[key], min_size=1, keys=[ "attributes", @@ -62,7 +64,7 @@ async def test_saas_access_request_task( ], ) - for item in v[f"{dataset_name}:events"]: + for item in v[key]: assert_rows_match( [item["attributes"]], min_size=1, @@ -74,5 +76,5 @@ async def test_saas_access_request_task( ], ) - for item in v[f"{dataset_name}:events"]: + for item in v[key]: assert datadog_identity_email in item["attributes"]["message"] From bf9f241f9e937c8f140772221d0e4f39b107d177 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 8 Sep 2022 15:23:03 -0700 Subject: [PATCH 2/3] Adding retry checks to access data seeding --- CHANGELOG.md | 5 ++- tests/ops/fixtures/saas/datadog_fixtures.py | 41 +++++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1dc6e099..cb28e7fcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,9 +37,9 @@ The types of changes are: * Wrap up the email connector - it sends an email with erasure instructions as part of request execution [#1246](https://github.com/ethyca/fidesops/pull/1246) * Mapping Vault environment variables in docker-compose.yml [#1275](https://github.com/ethyca/fidesops/pull/1275) * Foundations for a new "manual_webhook" connector type [#1267](https://github.com/ethyca/fidesops/pull/1267) -* Adds Fixture to Datadog Tests [1269](https://github.com/ethyca/fidesops/pull/1269) +* Data seeding for Datadog access tests [#1269](https://github.com/ethyca/fidesops/pull/1269) -### Docs +### Docs * Fix analytics opt out environment variable name [#1170](https://github.com/ethyca/fidesops/pull/1170) * Added how to view a subject request history and reprocess a subject request [#1164](https://github.com/ethyca/fidesops/pull/1164) @@ -53,6 +53,7 @@ The types of changes are: * Fix for pytest-asyncio bug [#1260](https://github.com/ethyca/fidesops/pull/1260) * Fix download link in privacy center [#1264](https://github.com/ethyca/fidesops/pull/1264) * Make admin ui work when volumes are mounted [#1266](https://github.com/ethyca/fidesops/pull/1266) +* Fixed typo in enum value [#1280](https://github.com/ethyca/fidesops/pull/1280) ## [1.7.2](https://github.com/ethyca/fidesops/compare/1.7.1...1.7.2) diff --git a/tests/ops/fixtures/saas/datadog_fixtures.py b/tests/ops/fixtures/saas/datadog_fixtures.py index f32ecd43e..ef5a3b057 100644 --- a/tests/ops/fixtures/saas/datadog_fixtures.py +++ b/tests/ops/fixtures/saas/datadog_fixtures.py @@ -16,6 +16,7 @@ load_config_with_replacement, load_dataset_with_replacement, ) +from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets secrets = get_secrets("datadog") @@ -100,24 +101,15 @@ def datadog_dataset_config( @pytest.fixture(scope="session") def datadog_access_data(datadog_secrets, datadog_identity_email): - url = "https://api.datadoghq.com/api/v2/logs/events?filter[from]=0&filter[query]=noonari@xyz.com&filter[to]=now&page[limit]=1000" - payload = {} - headers = { - "DD-API-KEY": f'{datadog_secrets.get("api_key")}', - "DD-APPLICATION-KEY": f'{datadog_secrets.get("app_key")}', - } - - response = requests.request("GET", url, headers=headers, data=payload) - - if datadog_identity_email not in response.text: + if not _logs_exist(datadog_identity_email, datadog_secrets): url = "https://http-intake.logs.datadoghq.com/api/v2/logs" payload = [ { "ddsource": "nginx", "ddtags": "env:staging,version:5.1", "hostname": "i-012345678", - "message": f"2019-11-19T14:37:58,995 INFO [process.name][20081{i}] Hello noonari@xyz.com", + "message": f"2019-11-19T14:37:58,995 INFO [process.name][20081{i}] Hello {datadog_identity_email}", "service": "payment", } for i in range(25) @@ -125,6 +117,31 @@ def datadog_access_data(datadog_secrets, datadog_identity_email): headers = { "Content-Type": "application/json", "Accept": "application/json", - "DD-API-KEY": f'{datadog_secrets.get("api_key")}', + "DD-API-KEY": datadog_secrets.get("api_key"), } requests.request("POST", url, headers=headers, json=payload) + + poll_for_existence( + _logs_exist, + (datadog_identity_email, datadog_secrets), + ) + + +def _logs_exist(datadog_identity_email: str, datadog_secrets): + url = "https://api.datadoghq.com/api/v2/logs/events" + params = { + "filter[from]": 0, + "filter[query]": datadog_identity_email, + "filter[to]": "now", + "page[limit]": 1000, + } + headers = { + "DD-API-KEY": datadog_secrets.get("api_key"), + "DD-APPLICATION-KEY": datadog_secrets.get("app_key"), + } + response = requests.request("GET", url, params=params, headers=headers) + + if datadog_identity_email not in response.text: + return None + + return response.json() From 0428a47730c5ea5e7af1432cc0a383088ea852ac Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 8 Sep 2022 15:27:00 -0700 Subject: [PATCH 3/3] Adding docstrings --- tests/ops/fixtures/saas/datadog_fixtures.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/ops/fixtures/saas/datadog_fixtures.py b/tests/ops/fixtures/saas/datadog_fixtures.py index ef5a3b057..31e4401d6 100644 --- a/tests/ops/fixtures/saas/datadog_fixtures.py +++ b/tests/ops/fixtures/saas/datadog_fixtures.py @@ -101,6 +101,10 @@ def datadog_dataset_config( @pytest.fixture(scope="session") def datadog_access_data(datadog_secrets, datadog_identity_email): + """ + Checks if logs exist for the identity email, logs are created if they + don't exist and we poll until the logs are present in the events endpoint + """ if not _logs_exist(datadog_identity_email, datadog_secrets): url = "https://http-intake.logs.datadoghq.com/api/v2/logs" @@ -128,6 +132,8 @@ def datadog_access_data(datadog_secrets, datadog_identity_email): def _logs_exist(datadog_identity_email: str, datadog_secrets): + """Checks if logs exist for the given identity email""" + url = "https://api.datadoghq.com/api/v2/logs/events" params = { "filter[from]": 0,