1385 saas connector twilio conversation erasure (#1673)

Co-authored-by: Hamza W <hamza@Hamzas-MacBook-Pro.local>
Co-authored-by: Adrian Galvan <adrian@ethyca.com>

CHANGELOG.md

@@ -27,6 +27,7 @@ The types of changes are:
 * Adds SMS message template for all subject notifications [#1743](https://github.com/ethyca/fides/pull/1743)
 * Privacy-Center-Cypress workflow for CI checks of the Privacy Center. [#1722](https://github.com/ethyca/fides/pull/1722)
 * Privacy Center `fides-consent.js` script for accessing consent on external pages. [Details](/clients/privacy-center/packages/fides-consent/README.md)
+* Erasure support for Twilio Conversations API [#1673](https://github.com/ethyca/fides/pull/1673)
 
 ### Changed
 

data/saas/config/twilio_conversations_config.yml

@@ -3,16 +3,13 @@ saas_config:
   name: Twilio Conversations SaaS Config
   type: twilio_conversations
   description: A sample schema representing the Twilio Conversations connector for Fidesops
-  version: 0.0.1
+  version: 0.0.2
 
   connector_params:
     - name: domain
       default_value: conversations.twilio.com
     - name: account_id
     - name: password
-    - name: page_size
-      default_value: 1000
-      description: The maximum number of records to return per page
 
   external_references:
     - name: twilio_user_id
@@ -45,14 +42,22 @@ saas_config:
             - name: sid
               references:
                 - twilio_user_id
+        update:
+          request_override: twilio_user_update
+          param_values:
+            - name: sid
+              references:
+                - dataset: <instance_fides_key>
+                  field: user.sid
+                  direction: from
     - name: user_conversations
       requests:
         read:
           method: GET
           path: /v1/Users/<user_id>/Conversations
           query_params:
             - name: PageSize
-              value: <page_size>
+              value: 1000
           param_values:
             - name: user_id
               references:
@@ -72,7 +77,7 @@ saas_config:
           path: /v1/Conversations/<conversation_id>/Messages
           query_params:
             - name: PageSize
-              value: <page_size>
+              value: 1000
           param_values:
             - name: conversation_id
               references:
@@ -85,14 +90,27 @@ saas_config:
               source: body
               path: meta.next_page_url
           data_path: messages
+        update:
+          request_override: twilio_conversation_message_update
+          param_values:
+            - name: conversation_id
+              references:
+                - dataset: <instance_fides_key>
+                  field: conversation_messages.conversation_sid
+                  direction: from
+            - name: message_id
+              references:
+                - dataset: <instance_fides_key>
+                  field: conversation_messages.sid
+                  direction: from
     - name: conversation_participants
       requests:
         read:
           method: GET
           path: /v1/Conversations/<conversation_id>/Participants
           query_params:
             - name: PageSize
-              value: <page_size>
+              value: 1000
           param_values:
             - name: conversation_id
               references:

data/saas/dataset/twilio_conversations_dataset.yml

@@ -41,6 +41,7 @@ dataset:
             data_categories: [system.operations]
             fidesops_meta:
               data_type: string
+              primary_key: true
           - name: attributes
             data_categories: [system.operations]
             fidesops_meta:
@@ -54,9 +55,11 @@ dataset:
             fidesops_meta:
               data_type: string
           - name: links
-            data_categories: [system.operations]
-            fidesops_meta:
-              data_type: string[]
+            fields:
+              - name: user_conversations
+                data_categories: [system.operations]
+                fidesops_meta:
+                  data_type: string
       - name: user_conversations
         fields:
           - name: notification_level
@@ -187,6 +190,7 @@ dataset:
             data_categories: [system.operations]
             fidesops_meta:
               data_type: string
+              primary_key: true
           - name: attributes
             data_categories: [system.operations]
             fidesops_meta:

src/fides/api/ops/service/saas_request/__init__.py

@@ -4,4 +4,5 @@
     domo_request_overrides,
     firebase_auth_request_overrides,
     mailchimp_request_overrides,
+    twilio_request_overrides,
 )

src/fides/api/ops/service/saas_request/override_implementations/twilio_request_overrides.py

@@ -0,0 +1,122 @@
+import logging
+from typing import Any, Dict, List
+
+import requests
+
+from fides.api.ops.common_exceptions import (
+    ClientUnsuccessfulException,
+    ConnectionException,
+)
+from fides.api.ops.models.policy import Policy
+from fides.api.ops.models.privacy_request import PrivacyRequest
+from fides.api.ops.service.saas_request.saas_request_override_factory import (
+    SaaSRequestType,
+    register,
+)
+from fides.api.ops.util.saas_util import to_pascal_case
+from fides.ctl.core.config import get_config
+
+CONFIG = get_config()
+logger = logging.getLogger(__name__)
+
+
+@register("twilio_user_update", [SaaSRequestType.UPDATE])
+def twilio_user_update(
+    param_values_per_row: List[Dict[str, Any]],
+    policy: Policy,
+    privacy_request: PrivacyRequest,
+    secrets: Dict[str, Any],
+) -> int:
+    rows_updated = 0
+
+    for row_param_values in param_values_per_row:
+        # get params to be used in update request
+        user_id = row_param_values.get("sid")
+
+        # check if the privacy_request targeted emails for erasure,
+        # if so rewrite with a format that can be accepted by Twilio
+        # regardless of the masking strategy in use
+        masked_object_fields = row_param_values["masked_object_fields"]
+
+        for k in masked_object_fields.copy().keys():
+            new_key = to_pascal_case(k)
+            masked_object_fields[new_key] = masked_object_fields.pop(k)
+
+        update_body = masked_object_fields
+
+        auth = secrets["account_id"], secrets["password"]
+
+        try:
+            response = requests.post(
+                url=f'https://{secrets["domain"]}/v1/Users/{user_id}',
+                auth=auth,
+                data=update_body,
+            )
+
+        # here we mimic the sort of error handling done in the core framework
+        # by the AuthenticatedClient. Extenders can chose to handle errors within
+        # their implementation as they wish.
+        except Exception as e:
+            if CONFIG.dev_mode:  # pylint: disable=R1720
+                raise ConnectionException(
+                    f"Operational Error connecting to Twilio Conversations API with error: {e}"
+                )
+            else:
+                raise ConnectionException("Operational Error connecting to Twilio Conversations API.")
+        if not response.ok:
+            raise ClientUnsuccessfulException(status_code=response.status_code)
+
+        rows_updated += 1
+    return rows_updated
+
+
+@register("twilio_conversation_message_update", [SaaSRequestType.UPDATE])
+def twilio_conversation_message_update(
+    param_values_per_row: List[Dict[str, Any]],
+    policy: Policy,
+    privacy_request: PrivacyRequest,
+    secrets: Dict[str, Any],
+) -> int:
+    rows_updated = 0
+
+    for row_param_values in param_values_per_row:
+        # get params to be used in update request
+        conversation_id = row_param_values.get("conversation_id")
+        message_id = row_param_values.get("message_id")
+
+        # check if the privacy_request targeted emails for erasure,
+        # if so rewrite with a format that can be accepted by Twilio
+        # regardless of the masking strategy in use
+        masked_object_fields = row_param_values["masked_object_fields"]
+
+        for k in masked_object_fields.copy().keys():
+            new_key = to_pascal_case(k)
+            masked_object_fields[new_key] = masked_object_fields.pop(k)
+
+        update_body = masked_object_fields
+
+        auth = secrets["account_id"], secrets["password"]
+        try:
+            response = requests.post(
+                url=f'https://{secrets["domain"]}/v1/Conversations/{conversation_id}/Messages/{message_id}',
+                auth=auth,
+                data=update_body,
+            )
+
+        # here we mimic the sort of error handling done in the core framework
+        # by the AuthenticatedClient. Extenders can chose to handle errors within
+        # their implementation as they wish.
+        except Exception as e:
+            if CONFIG.dev_mode:  # pylint: disable=R1720
+                raise ConnectionException(
+                    f"Operational Error connecting to Twilio Conversations API with error: {e}"
+                )
+            else:
+                raise ConnectionException(
+                    "Operational Error connecting to Twilio Conversations API."
+                )
+        if not response.ok:
+            raise ClientUnsuccessfulException(status_code=response.status_code)
+
+        rows_updated += 1
+    return rows_updated

src/fides/api/ops/util/saas_util.py

@@ -299,3 +299,9 @@ def encode_file_contents(file_path: str) -> str:
     file_path = load_file([file_path])
     with open(file_path, "rb") as file:
         return bytes_to_b64_str(file.read())
+
+
+def to_pascal_case(s: str) -> str:
+    s = s.title()
+    s = s.replace("_", "")
+    return s