Merge branch 'main' into allow-no-content-response

CHANGELOG.md

@@ -18,12 +18,14 @@ The types of changes are:
 ## [Unreleased](https://github.com/ethyca/fides/compare/2.35.0...main)
 
 ### Added
+- Added access and erasure support for Marigold Engage by Sailthru integration [#4826](https://github.com/ethyca/fides/pull/4826)
 - Added multiple language translations support for privacy center consent page [#4785](https://github.com/ethyca/fides/pull/4785)
 - Added ability to export the contents of datamap report [#1545](https://ethyca.atlassian.net/browse/PROD-1545)
 
 ### Fixed
 - Remove the extra 'white-space: normal' CSS for FidesJS HTML descriptions [#4850](https://github.com/ethyca/fides/pull/4850)
 
+
 ## [2.35.0](https://github.com/ethyca/fides/compare/2.34.0...2.35.0)
 
 ### Added

data/saas/config/marigold_engage_config.yml

@@ -0,0 +1,37 @@
+saas_config:
+  fides_key: <instance_fides_key>
+  name: Marigold Engage by Sailthru
+  type: marigold_engage
+  description: A sample schema representing the Marigold Engage via Sailthru connector for Fides
+  version: 0.1.0
+
+  connector_params:
+    - name: domain
+      default_value: api.sailthru.com
+    - name: api_key
+      label: API key
+      sensitive: True
+    - name: secret
+      label: Marigold secret
+      sensitive: True
+
+  client_config:
+    protocol: https
+    host: <domain>
+
+  test_request:
+    request_override: marigold_engage_test
+
+  endpoints:
+    - name: user
+      requests:
+        read:
+          request_override: marigold_engage_user_read
+          param_values:
+            - name: email
+              identity: email
+        delete:
+          request_override: marigold_engage_user_delete
+          param_values:
+            - name: email
+              identity: email

data/saas/dataset/marigold_engage_dataset.yml

@@ -0,0 +1,47 @@
+dataset:
+  - fides_key: <instance_fides_key>
+    name: Marigold Engage Dataset
+    description: A sample dataset representing the Marigold Engage integration for Fides
+    collections:
+      - name: user
+        fields:
+          - name: activity
+            fidesops_meta:
+              data_type: object
+            fields:
+              - name: create_time
+                data_categories: [system.operations]
+                fidesops_meta:
+                  data_type: string
+          - name: engagement
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: keys
+            fidesops_meta:
+              data_type: object
+            fields:
+              - name: sid
+                data_categories: [system.operations]
+                fidesops_meta:
+                  data_type: string
+              - name: cookie
+                data_categories: [user.device.cookie_id]
+                fidesops_meta:
+                  data_type: string
+              - name: email
+                data_categories: [user.contact.email]
+                fidesops_meta:
+                  primary_key: True
+                  data_type: string
+          - name: lists
+          - name: optout_email
+            data_categories: [system.operations]
+            fidesops_meta:
+              data_type: string
+          - name: smart_lists
+          - name: vars
+          - name: purchases
+          - name: device
+          - name: purchase_incomplete
+          - name: lifetime

docker-compose.integration-tests.yml

@@ -25,7 +25,7 @@ services:
   mysql-test:
     image: mysql:8
     platform: linux/amd64
-    command: --default-authentication-plugin=mysql_native_password
+    command: --mysql-native-password=ON
     restart: always
     environment:
       - MYSQL_HOST=mysql_example

docker/docker-compose.integration-mysql.yml

@@ -2,7 +2,7 @@ services:
   mysql_example:
     image: mysql
     platform: linux/amd64
-    command: --default-authentication-plugin=mysql_native_password
+    command: --mysql-native-password=ON
     restart: always
     environment:
       - MYSQL_HOST=mysql_example

src/fides/api/service/connectors/saas_connector.py

@@ -145,14 +145,22 @@ def test_connection(self) -> Optional[ConnectionTestStatus]:
         """Generates and executes a test connection based on the SaaS config"""
         test_request: SaaSRequest = self.saas_config.test_request
         self.set_saas_request_state(test_request)
-        prepared_request = map_param_values(
-            "test",
-            f"{self.configuration.name}",
-            test_request,
-            self.secrets,
-        )
         client: AuthenticatedClient = self.create_client()
-        client.send(prepared_request, test_request.ignore_errors)
+
+        if test_request.request_override:
+            self._invoke_test_request_override(
+                test_request.request_override,
+                client,
+                self.secrets,
+            )
+        else:
+            prepared_request = map_param_values(
+                "test",
+                f"{self.configuration.name}",
+                test_request,
+                self.secrets,
+            )
+            client.send(prepared_request, test_request.ignore_errors)
         self.unset_connector_state()
         return ConnectionTestStatus.succeeded
 
@@ -611,6 +619,35 @@ def _unwrap_response_data(saas_request: SaaSRequest, response: Response) -> Any:
                 f"Unable to parse JSON response from {saas_request.path}"
             )
 
+    @staticmethod
+    def _invoke_test_request_override(
+        override_function_name: str,
+        client: AuthenticatedClient,
+        secrets: Any,
+    ) -> List[Row]:
+        """
+        Invokes the appropriate user-defined SaaS request override for a test request.
+
+        Contains error handling for uncaught exceptions coming out of the override.
+        """
+        override_function: Callable[..., Union[List[Row], int, None]] = (
+            SaaSRequestOverrideFactory.get_override(
+                override_function_name, SaaSRequestType.TEST
+            )
+        )
+        try:
+            return override_function(
+                client,
+                secrets,
+            )  # type: ignore
+        except Exception as exc:
+            logger.error(
+                "Encountered error executing override test function '{}'",
+                override_function_name,
+                exc_info=True,
+            )
+            raise FidesopsException(str(exc))
+
     @staticmethod
     def _invoke_read_request_override(
         override_function_name: str,
@@ -626,7 +663,7 @@ def _invoke_read_request_override(
 
         Contains error handling for uncaught exceptions coming out of the override.
         """
-        override_function: Callable[..., Union[List[Row], int]] = (
+        override_function: Callable[..., Union[List[Row], int, None]] = (
             SaaSRequestOverrideFactory.get_override(
                 override_function_name, SaaSRequestType.READ
             )
@@ -666,7 +703,7 @@ def _invoke_masking_request_override(
         Includes the necessary data preparations for override input
         and has error handling for uncaught exceptions coming out of the override
         """
-        override_function: Callable[..., Union[List[Row], int]] = (
+        override_function: Callable[..., Union[List[Row], int, None]] = (
             SaaSRequestOverrideFactory.get_override(
                 override_function_name, SaaSRequestType(query_config.action)
             )

src/fides/api/service/saas_request/override_implementations/marigold_engage_request_overrides.py

@@ -0,0 +1,123 @@
+import hashlib
+import json
+from typing import Any, Dict, List
+
+from fides.api.graph.traversal import TraversalNode
+from fides.api.models.policy import Policy
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams
+from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient
+from fides.api.service.saas_request.saas_request_override_factory import (
+    SaaSRequestType,
+    register,
+)
+from fides.api.util.collection_util import Row
+
+
+@register("marigold_engage_test", [SaaSRequestType.TEST])
+def marigold_engage_test(client: AuthenticatedClient, secrets: Dict[str, Any]) -> None:
+    """
+    Calls Marigold Engage's `GET /list` endpoint with a signed payload.
+    """
+
+    client.send(
+        SaaSRequestParams(
+            method=HTTPMethod.GET,
+            path="/list",
+            query_params=signed_payload(secrets, {}),
+        )
+    )
+
+
+@register("marigold_engage_user_read", [SaaSRequestType.READ])
+def marigold_engage_user_read(
+    client: AuthenticatedClient,
+    node: TraversalNode,
+    policy: Policy,
+    privacy_request: PrivacyRequest,
+    input_data: Dict[str, List[Any]],
+    secrets: Dict[str, Any],
+) -> List[Row]:
+    """
+    Calls Marigold Engage's `GET /user` endpoint with a signed payload.
+    """
+
+    output = []
+    emails = input_data.get("email", [])
+    for email in emails:
+        payload = {
+            "id": email,
+            "key": "email",
+            "fields": {
+                "activity": 1,
+                "engagement": 1,
+                "keys": 1,
+                "lists": 1,
+                "optout_email": 1,
+                "smart_lists": 1,
+                "vars": 1,
+                "purchases": 1,
+                "device": 1,
+                "purchase_incomplete": 1,
+                "lifetime": 1,
+            },
+        }
+        response = client.send(
+            SaaSRequestParams(
+                method=HTTPMethod.GET,
+                path="/user",
+                query_params=signed_payload(secrets, payload),
+            )
+        )
+        user = response.json()
+        output.append(user)
+
+    return output
+
+
+@register("marigold_engage_user_delete", [SaaSRequestType.DELETE])
+def marigold_engage_user_delete(
+    client: AuthenticatedClient,
+    param_values_per_row: List[Dict[str, Any]],
+    policy: Policy,
+    privacy_request: PrivacyRequest,
+    secrets: Dict[str, Any],
+) -> int:
+    """
+    Calls Marigold Engage's `DELETE /user` endpoint with a signed payload.
+    """
+
+    rows_deleted = 0
+    for row_param_values in param_values_per_row:
+        email = row_param_values["email"]
+        client.send(
+            SaaSRequestParams(
+                method=HTTPMethod.DELETE,
+                path="/user",
+                query_params=signed_payload(secrets, {"id": email}),
+            )
+        )
+        rows_deleted += 1
+    return rows_deleted
+
+
+def signed_payload(secrets: Dict[str, Any], payload: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Creates a signed payload dictionary with an MD5 hash of the secret, API key, format, and payload.
+    """
+
+    # the signature is the md5 hash of the concatenated string
+    # of secret, API key, format, and stringified payload
+    stringified_payload = json.dumps(payload)
+    parameter_values = (
+        f'{secrets["secret"]}{secrets["api_key"]}json{stringified_payload}'
+    )
+    hash_value = hashlib.md5(parameter_values.encode())
+    sig = hash_value.hexdigest()
+
+    return {
+        "api_key": secrets["api_key"],
+        "sig": sig,
+        "format": "json",
+        "json": stringified_payload,
+    }