Added fixture to create a new data everytime test runs.

CHANGELOG.md

@@ -37,6 +37,7 @@ The types of changes are:
 * Wrap up the email connector - it sends an email with erasure instructions as part of request execution [#1246](https://github.com/ethyca/fidesops/pull/1246)
 * Mapping Vault environment variables in docker-compose.yml [#1275](https://github.com/ethyca/fidesops/pull/1275)
 * Foundations for a new "manual_webhook" connector type [#1267](https://github.com/ethyca/fidesops/pull/1267)
+* Data seeding for Datadog access tests [#1269](https://github.com/ethyca/fidesops/pull/1269)
 
 ### Docs
 

tests/ops/fixtures/saas/datadog_fixtures.py

@@ -2,6 +2,7 @@
 
 import pydash
 import pytest
+import requests
 from fideslib.db import session
 from sqlalchemy.orm import Session
 
@@ -15,6 +16,7 @@
     load_config_with_replacement,
     load_dataset_with_replacement,
 )
+from tests.ops.test_helpers.saas_test_utils import poll_for_existence
 from tests.ops.test_helpers.vault_client import get_secrets
 
 secrets = get_secrets("datadog")
@@ -95,3 +97,57 @@ def datadog_dataset_config(
     )
     yield dataset
     dataset.delete(db=db)
+
+
+@pytest.fixture(scope="session")
+def datadog_access_data(datadog_secrets, datadog_identity_email):
+    """
+    Checks if logs exist for the identity email, logs are created if they
+    don't exist and we poll until the logs are present in the events endpoint
+    """
+
+    if not _logs_exist(datadog_identity_email, datadog_secrets):
+        url = "https://http-intake.logs.datadoghq.com/api/v2/logs"
+        payload = [
+            {
+                "ddsource": "nginx",
+                "ddtags": "env:staging,version:5.1",
+                "hostname": "i-012345678",
+                "message": f"2019-11-19T14:37:58,995 INFO [process.name][20081{i}] Hello {datadog_identity_email}",
+                "service": "payment",
+            }
+            for i in range(25)
+        ]
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "DD-API-KEY": datadog_secrets.get("api_key"),
+        }
+        requests.request("POST", url, headers=headers, json=payload)
+
+        poll_for_existence(
+            _logs_exist,
+            (datadog_identity_email, datadog_secrets),
+        )
+
+
+def _logs_exist(datadog_identity_email: str, datadog_secrets):
+    """Checks if logs exist for the given identity email"""
+
+    url = "https://api.datadoghq.com/api/v2/logs/events"
+    params = {
+        "filter[from]": 0,
+        "filter[query]": datadog_identity_email,
+        "filter[to]": "now",
+        "page[limit]": 1000,
+    }
+    headers = {
+        "DD-API-KEY": datadog_secrets.get("api_key"),
+        "DD-APPLICATION-KEY": datadog_secrets.get("app_key"),
+    }
+    response = requests.request("GET", url, params=params, headers=headers)
+
+    if datadog_identity_email not in response.text:
+        return None
+
+    return response.json()

tests/ops/integration_tests/saas/test_datadog_task.py

@@ -28,6 +28,7 @@ async def test_saas_access_request_task(
     datadog_connection_config,
     datadog_dataset_config,
     datadog_identity_email,
+    datadog_access_data,
 ) -> None:
     """Full access request based on the Datadog SaaS config"""
 
@@ -51,9 +52,10 @@ async def test_saas_access_request_task(
         {"email": datadog_identity_email},
         db,
     )
+    key = f"{dataset_name}:events"
 
     assert_rows_match(
-        v[f"{dataset_name}:events"],
+        v[key],
         min_size=1,
         keys=[
             "attributes",
@@ -62,7 +64,7 @@ async def test_saas_access_request_task(
         ],
     )
 
-    for item in v[f"{dataset_name}:events"]:
+    for item in v[key]:
         assert_rows_match(
             [item["attributes"]],
             min_size=1,
@@ -74,5 +76,5 @@ async def test_saas_access_request_task(
             ],
         )
 
-    for item in v[f"{dataset_name}:events"]:
+    for item in v[key]:
         assert datadog_identity_email in item["attributes"]["message"]