Merge pull request #1994 from openedx/hamza/ENT-8276-record-corner-st…

…one-api-calls

feat: update cornerstone client to store API calls in DB

CHANGELOG.rst

@@ -16,6 +16,10 @@ Change Log
 Unreleased
 ----------
 
+[4.11.3]
+---------
+* feat: update cornerstone client to store API calls in DB
+
 [4.11.2]
 ---------
 * feat: added caching for fetching degreed course id 

enterprise/__init__.py

@@ -2,4 +2,4 @@
 Your project description goes here.
 """
 
-__version__ = "4.11.2"
+__version__ = "4.11.3"

integrated_channels/cornerstone/client.py

@@ -5,6 +5,7 @@
 import base64
 import json
 import logging
+import time
 
 import requests
 
@@ -87,6 +88,9 @@ def create_course_completion(self, user_id, payload):
         Raises:
             HTTPError: if we received a failure response code from Cornerstone
         """
+        IntegratedChannelAPIRequestLogs = apps.get_model(
+            "integrated_channel", "IntegratedChannelAPIRequestLogs"
+        )
         json_payload = json.loads(payload)
         callback_url = json_payload['data'].pop('callbackUrl')
         session_token = self.enterprise_configuration.session_token
@@ -104,7 +108,7 @@ def create_course_completion(self, user_id, payload):
             completion_path=self.global_cornerstone_config.completion_status_api_path,
             session_token=session_token,
         )
-
+        start_time = time.time()
         response = requests.post(
             url,
             json=[json_payload['data']],
@@ -113,6 +117,16 @@ def create_course_completion(self, user_id, payload):
                 'Content-Type': 'application/json'
             }
         )
+        duration_seconds = time.time() - start_time
+        IntegratedChannelAPIRequestLogs.store_api_call(
+            enterprise_customer=self.enterprise_configuration.enterprise_customer,
+            enterprise_customer_configuration_id=self.enterprise_configuration.id,
+            endpoint=url,
+            payload=json.dumps(json_payload["data"]),
+            time_taken=duration_seconds,
+            status_code=response.status_code,
+            response_body=response.text,
+        )
         return response.status_code, response.text
 
     def create_assessment_reporting(self, user_id, payload):

integrated_channels/cornerstone/migrations/0032_delete_cornerstoneapirequestlogs.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.2.22 on 2024-01-25 09:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cornerstone', '0031_cornerstoneapirequestlogs'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='CornerstoneAPIRequestLogs',
+        ),
+    ]

integrated_channels/cornerstone/models.py

@@ -19,7 +19,6 @@
 from integrated_channels.cornerstone.transmitters.learner_data import CornerstoneLearnerTransmitter
 from integrated_channels.integrated_channel.models import (
     EnterpriseCustomerPluginConfiguration,
-    IntegratedChannelAPIRequestLogs,
     LearnerDataTransmissionAudit,
 )
 from integrated_channels.utils import is_valid_url
@@ -319,36 +318,3 @@ class CornerstoneCourseKey(models.Model):
 
     class Meta:
         app_label = 'cornerstone'
-
-
-class CornerstoneAPIRequestLogs(IntegratedChannelAPIRequestLogs):
-    """
-     A model to track basic information about every API call we make from the integrated channels.
-    """
-    user_agent = models.CharField(max_length=255)
-    user_ip = models.GenericIPAddressField(blank=True, null=True)
-
-    class Meta:
-        app_label = 'cornerstone'
-
-    def __str__(self):
-        """
-        Return a human-readable string representation of the object.
-        """
-        return (
-            f'<CornerstoneAPIRequestLogs {self.id}'
-            f' for enterprise customer {self.enterprise_customer}, '
-            f', enterprise_customer_configuration_id: {self.enterprise_customer_configuration_id}>'
-            f', endpoint: {self.endpoint}'
-            f', time_taken: {self.time_taken}'
-            f', user_agent: {self.user_agent}'
-            f', user_ip: {self.user_ip}'
-            f', api_record.body: {self.api_record.body}'
-            f', api_record.status_code: {self.api_record.status_code}'
-        )
-
-    def __repr__(self):
-        """
-        Return uniquely identifying string representation.
-        """
-        return self.__str__()

integrated_channels/cornerstone/views.py

@@ -3,6 +3,8 @@
 """
 
 import datetime
+import json
+import time
 from logging import getLogger
 
 from dateutil import parser
@@ -11,6 +13,7 @@
 from rest_framework.authentication import SessionAuthentication
 from rest_framework.response import Response
 
+from django.apps import apps
 from django.utils.http import parse_http_date_safe
 
 from enterprise.api.throttles import ServiceUserThrottle
@@ -98,7 +101,11 @@ class CornerstoneCoursesListView(BaseViewSet):
     """
 
     def get(self, request, *args, **kwargs):
+        start_time = time.time()
         enterprise_customer_uuid = request.GET.get('ciid')
+        IntegratedChannelAPIRequestLogs = apps.get_model(
+            "integrated_channel", "IntegratedChannelAPIRequestLogs"
+        )
         if not enterprise_customer_uuid:
             return Response(
                 status=status.HTTP_400_BAD_REQUEST,
@@ -156,12 +163,16 @@ def get(self, request, *args, **kwargs):
                                 f'fixing modified/header mismatch')
                     if_modified_since_dt = datetime.datetime.fromtimestamp(if_modified_since)
                     item['LastModifiedUTC'] = if_modified_since_dt.strftime(ISO_8601_DATE_FORMAT)
-        # TODO remove following logs (temporarily added)
-        logger.info(
-            f"[Cornerstone]: request.headers={request.headers}"
-            f"GET params={request.GET}"
-            f"enterprise_config={enterprise_config}"
-            f"enterprise_config.id={enterprise_config.id}"
-            f"data={data}"
+        duration_seconds = time.time() - start_time
+        headers_dict = dict(request.headers)
+        headers_json = json.dumps(headers_dict)
+        IntegratedChannelAPIRequestLogs.store_api_call(
+            enterprise_customer=enterprise_customer,
+            enterprise_customer_configuration_id=enterprise_config.id,
+            endpoint=request.get_full_path(),
+            payload=f"Request Headers: {headers_json}",
+            time_taken=duration_seconds,
+            status_code=200,
+            response_body=json.dumps(data)
         )
         return Response(data)

integrated_channels/integrated_channel/admin/__init__.py

@@ -4,7 +4,11 @@
 
 from django.contrib import admin
 
-from integrated_channels.integrated_channel.models import ApiResponseRecord, ContentMetadataItemTransmission
+from integrated_channels.integrated_channel.models import (
+    ApiResponseRecord,
+    ContentMetadataItemTransmission,
+    IntegratedChannelAPIRequestLogs,
+)
 from integrated_channels.utils import get_enterprise_customer_from_enterprise_enrollment
 
 

integrated_channels/integrated_channel/migrations/0031_alter_integratedchannelapirequestlogs_options.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.2.22 on 2024-01-25 09:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('integrated_channel', '0030_integratedchannelapirequestlogs'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='integratedchannelapirequestlogs',
+            options={'verbose_name_plural': 'Integrated channels API request logs'},
+        ),
+        migrations.RemoveField(
+            model_name='integratedchannelapirequestlogs',
+            name='api_record',
+        ),
+        migrations.AddField(
+            model_name='integratedchannelapirequestlogs',
+            name='response_body',
+            field=models.TextField(blank=True, help_text='API call response body', null=True),
+        ),
+        migrations.AddField(
+            model_name='integratedchannelapirequestlogs',
+            name='status_code',
+            field=models.PositiveIntegerField(blank=True, help_text='API call response HTTP status code', null=True),
+        ),
+        migrations.AlterField(
+            model_name='integratedchannelapirequestlogs',
+            name='time_taken',
+            field=models.FloatField(),
+        ),
+    ]

integrated_channels/integrated_channel/migrations/0032_alter_integratedchannelapirequestlogs_endpoint.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.22 on 2024-01-29 13:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('integrated_channel', '0031_alter_integratedchannelapirequestlogs_options'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='integratedchannelapirequestlogs',
+            name='endpoint',
+            field=models.URLField(max_length=255),
+        ),
+    ]

integrated_channels/integrated_channel/models.py

@@ -883,43 +883,88 @@ class Meta:
 
 class IntegratedChannelAPIRequestLogs(TimeStampedModel):
     """
-     A model to track basic information about every API call we make from the integrated channels.
+    A model to track basic information about every API call we make from the integrated channels.
     """
+
     enterprise_customer = models.ForeignKey(
-        EnterpriseCustomer, on_delete=models.CASCADE)
+        EnterpriseCustomer, on_delete=models.CASCADE
+    )
     enterprise_customer_configuration_id = models.IntegerField(
-        blank=False, null=False, help_text='ID from the EnterpriseCustomerConfiguration model')
-    endpoint = models.TextField(blank=False, null=False)
+        blank=False,
+        null=False,
+        help_text="ID from the EnterpriseCustomerConfiguration model",
+    )
+    endpoint = models.URLField(
+        blank=False,
+        max_length=255,
+        null=False,
+    )
     payload = models.TextField(blank=False, null=False)
-    time_taken = models.DurationField(blank=False, null=False)
-    api_record = models.OneToOneField(
-        ApiResponseRecord,
-        blank=True,
-        null=True,
-        on_delete=models.CASCADE,
-        help_text=_(
-            'Data pertaining to the transmissions API request response.')
+    time_taken = models.FloatField(blank=False, null=False)
+    status_code = models.PositiveIntegerField(
+        help_text="API call response HTTP status code", blank=True, null=True
+    )
+    response_body = models.TextField(
+        help_text="API call response body", blank=True, null=True
     )
 
     class Meta:
-        app_label = 'integrated_channel'
+        app_label = "integrated_channel"
+        verbose_name_plural = "Integrated channels API request logs"
 
     def __str__(self):
         """
         Return a human-readable string representation of the object.
         """
         return (
-            f'<IntegratedChannelAPIRequestLog {self.id}'
-            f' for enterprise customer {self.enterprise_customer}, '
-            f', enterprise_customer_configuration_id: {self.enterprise_customer_configuration_id}>'
-            f', endpoint: {self.endpoint}'
-            f', time_taken: {self.time_taken}'
-            f', api_record.body: {self.api_record.body}'
-            f', api_record.status_code: {self.api_record.status_code}'
+            f"<IntegratedChannelAPIRequestLog {self.id}"
+            f" for enterprise customer {self.enterprise_customer} "
+            f", enterprise_customer_configuration_id: {self.enterprise_customer_configuration_id}>"
+            f", endpoint: {self.endpoint}"
+            f", time_taken: {self.time_taken}"
+            f", response_body: {self.response_body}"
+            f", status_code: {self.status_code}"
         )
 
     def __repr__(self):
         """
         Return uniquely identifying string representation.
         """
         return self.__str__()
+
+    @classmethod
+    def store_api_call(
+        cls,
+        enterprise_customer,
+        enterprise_customer_configuration_id,
+        endpoint,
+        payload,
+        time_taken,
+        status_code,
+        response_body,
+    ):
+        """
+        Creates new record in IntegratedChannelAPIRequestLogs table.
+        """
+        try:
+            record = cls(
+                enterprise_customer=enterprise_customer,
+                enterprise_customer_configuration_id=enterprise_customer_configuration_id,
+                endpoint=endpoint,
+                payload=payload,
+                time_taken=time_taken,
+                status_code=status_code,
+                response_body=response_body,
+            )
+            record.save()
+        except Exception as e:  # pylint: disable=broad-except
+            LOGGER.error(
+                f"store_api_call raised error while storing API call: {e}"
+                f"enterprise_customer={enterprise_customer}"
+                f"enterprise_customer_configuration_id={enterprise_customer_configuration_id},"
+                f"endpoint={endpoint}"
+                f"payload={payload}"
+                f"time_taken={time_taken}"
+                f"status_code={status_code}"
+                f"response_body={response_body}"
+            )