From d5f9cb46d9f78f08db1cbffbd1a0029fa298ec54 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 18 Oct 2023 16:24:01 +0200 Subject: [PATCH 01/38] fix importing local settings for docker settings and refine compose file --- docker-compose.yaml | 2 -- elixir_daisy/settings_compose.py | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 16395d7c..e8653ec3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,8 +12,6 @@ services: - statics:/static - solrdata:/solr - .:/code - ports: - - "5000:5000" depends_on: - db - solr diff --git a/elixir_daisy/settings_compose.py b/elixir_daisy/settings_compose.py index d8e26819..92aa66b9 100644 --- a/elixir_daisy/settings_compose.py +++ b/elixir_daisy/settings_compose.py @@ -71,6 +71,10 @@ TESTING = os.environ.get("TEST", False) # import also local settings +try: + from .settings_local import * +except ImportError as e: + pass try: if TESTING: from .settings_ci import * From c9fff05b67635a0e6bce56a0f49b1e90e6bee8fb Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 19 Oct 2023 16:44:20 +0200 Subject: [PATCH 02/38] refine celery config and create the task definition --- elixir_daisy/celery_app.py | 10 ++- elixir_daisy/settings.py | 9 ++- notification/tasks.py | 161 +++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 notification/tasks.py diff --git a/elixir_daisy/celery_app.py b/elixir_daisy/celery_app.py index daa6ffd6..73c1e2cf 100644 --- a/elixir_daisy/celery_app.py +++ b/elixir_daisy/celery_app.py @@ -2,10 +2,16 @@ from celery import Celery -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "elixir_daisy.settings") - app = Celery("daisy") +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +# The uppercase name-space means that all Celery configuration options must +# be specified in uppercase instead of lowercase, and start with CELERY_, +# so for example the task_always_eager setting becomes CELERY_TASK_ALWAYS_EAGER, + app.config_from_object("django.conf:settings", namespace="CELERY") # Load task modules from all registered Django app configs. diff --git a/elixir_daisy/settings.py b/elixir_daisy/settings.py index e8cf869e..faf6cf34 100644 --- a/elixir_daisy/settings.py +++ b/elixir_daisy/settings.py @@ -144,16 +144,17 @@ # https://docs.djangoproject.com/en/2.0/topics/i18n/ LANGUAGE_CODE = "en-us" - TIME_ZONE = "Europe/Luxembourg" TZINFO = pytz.timezone(TIME_ZONE) - USE_I18N = True - USE_L10N = True - USE_TZ = True +# Celery configs + +CELERY_BROKER_URL = "amqp://guest:guest@localhost:5672//" +CELERY_TIMEZONE = "Europe/Luxembourg" + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ STATIC_URL = "/static/" diff --git a/notification/tasks.py b/notification/tasks.py new file mode 100644 index 00000000..560d54da --- /dev/null +++ b/notification/tasks.py @@ -0,0 +1,161 @@ +import datetime +from collections import defaultdict +from datetime import timedelta + +from celery import shared_task +from django.conf import settings +from django.utils import timezone +from django.db.models import Q + +from core.models import Contract, DataDeclaration, Dataset, Document, Project, User +from notification.email_sender import send_the_email +from notification.models import Notification, NotificationStyle, NotificationVerb + +# map each notification style to a delta +# delta correspond to the interval + a small delta +NOTIFICATION_MAPPING = { + NotificationStyle.once_per_day: timedelta(days=1, hours=8), + NotificationStyle.once_per_week: timedelta(days=7, hours=16), + NotificationStyle.once_per_month: timedelta(days=33), +} + +@shared_task +def create_notifications_for_entities(): + """ + Loops Through all the entitie that implement the Notificaiton Mixin + and creates a notification for each one of them according to the logic. + """ + now = timezone.now() + + pass + +@shared_task +def send_notifications_for_user_by_time(user_id, time): + """ + Send a notification report for the current user from the date to the most recent. + """ + # get latest notifications for the user, grouped by verb + notifications = Notification.objects.filter(actor__pk=user_id, time__gte=time) + + if not notifications: + return + + # group notifications per verb + notifications_by_verb = defaultdict(list) + for notif in notifications: + notifications_by_verb[notif.verb].append(notif) + + user = User.objects.get(pk=user_id) + context = {"time": time, "user": user, "notifications": dict(notifications_by_verb)} + send_the_email( + settings.EMAIL_DONOTREPLY, + user.email, + "Notifications", + "notification/notification_report", + context, + ) + + +@shared_task +def send_dataset_notification_for_user(user_id, dataset_id, created): + """ + Send the notification that a dataset have been updated. + """ + dataset = Dataset.objects.get(pk=dataset_id) + user = User.objects.get(pk=user_id) + context = {"user": user, "dataset": dataset, "created": created} + send_the_email( + settings.EMAIL_DONOTREPLY, + user.email, + "Notifications", + "notification/dataset_notification", + context, + ) + + +@shared_task +def send_notifications(period): + """ + Send notifications for each user based on the period selected. + Period must be one of `NotificationStyle` but 'every_time'. + """ + notification_style = NotificationStyle[period] + if notification_style is NotificationStyle.every_time: + raise KeyError("Key not permitted") + # get users with this setting + users = User.objects.filter( + notification_setting__style=notification_style + ).distinct() + # map the setting to a timeperiod + now = timezone.now() + time = now - NOTIFICATION_MAPPING[notification_style] + + # get latest notifications + users = users.filter(notifications__time__gte=time) + for user in users: + send_notifications_for_user_by_time.delay(user.id, time) + return users + + +@shared_task +def data_storage_expiry_notifications(): + now = timezone.now() + + # the user will receive notifications on two consecutive days prior to storage end date + window_2_start = now + datetime.timedelta(days=1) + window_2_end = now + datetime.timedelta(days=2) + + # the user will receive notifications on two consecutive days, two months prior to storage end date + window_60_start = now + datetime.timedelta(days=59) + window_60_end = now + datetime.timedelta(days=60) + + data_declarations = DataDeclaration.objects.filter( + Q( + end_of_storage_duration__gte=window_60_start, + end_of_storage_duration__lte=window_60_end, + ) + | Q( + end_of_storage_duration__gte=window_2_start, + end_of_storage_duration__lte=window_2_end, + ) + ).order_by("end_of_storage_duration") + + for ddec in data_declarations: + for custodian in ddec.dataset.local_custodians.all(): + Notification.objects.create( + actor=custodian, + verb=NotificationVerb.data_storage_expiry, + content_object=ddec, + ) + + +@shared_task +def document_expiry_notifications(): + now = timezone.now() + + # the user will receive notifications on two consecutive days prior to storage end date + window_2_start = now + datetime.timedelta(days=1) + window_2_end = now + datetime.timedelta(days=2) + + # the user will receive notifications on two consecutive days, two months prior to storage end date + window_60_start = now + datetime.timedelta(days=59) + window_60_end = now + datetime.timedelta(days=60) + + documents = Document.objects.filter( + Q(expiry_date__gte=window_60_start, expiry_date__lte=window_60_end) + | Q(expiry_date__gte=window_2_start, expiry_date__lte=window_2_end) + ).order_by("expiry_date") + + for document in documents: + print(document.content_type) + if str(document.content_type) == "project": + obj = Project.objects.get(pk=document.object_id) + if str(document.content_type) == "contract": + obj = Contract.objects.get(pk=document.object_id) + if obj: + for custodian in obj.local_custodians.all(): + Notification.objects.create( + actor=custodian, + verb=NotificationVerb.document_expiry, + content_object=obj, + ) From 10a6e45f3438680ceeee5438b87b561d93055c7c Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Fri, 20 Oct 2023 11:58:24 +0200 Subject: [PATCH 03/38] add NotifyMixin --- notification/__init__.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/notification/__init__.py b/notification/__init__.py index e69de29b..56f53fe2 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -0,0 +1,26 @@ +class NotifyMixin: + def get_notification_recipients(self): + """ + Should Query the users based on their notification settings + and the entity. + + Raises: + NotImplementedError: It should be implemented by the subclass + """ + raise NotImplementedError(f"Subclasses of {NotifyMixin.__name__} must implement {self.get_notification_recipients.__name__}") + + def make_notification(self): + """ + Creates a notifications for the reciepients based on + the business logic of the entity. + + Raises: + NotImplementedError: It should be implemented by the subclass + """ + raise NotImplementedError(f"Subclasses of {NotifyMixin.__name__} must implement {self.make_notification.__name__}") + + def get_absolute_url(self): + """ + Returns the absolute url of the entity. + """ + return None \ No newline at end of file From 43fc50e9e144efa8231a2e32fd6d43b0d31b0c74 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 23 Oct 2023 14:02:30 +0200 Subject: [PATCH 04/38] apply black linter --- elixir_daisy/celery_app.py | 2 +- notification/__init__.py | 18 +++++++++++------- notification/tasks.py | 4 ++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/elixir_daisy/celery_app.py b/elixir_daisy/celery_app.py index 73c1e2cf..790e64f5 100644 --- a/elixir_daisy/celery_app.py +++ b/elixir_daisy/celery_app.py @@ -9,7 +9,7 @@ # - namespace='CELERY' means all celery-related configuration keys # should have a `CELERY_` prefix. # The uppercase name-space means that all Celery configuration options must -# be specified in uppercase instead of lowercase, and start with CELERY_, +# be specified in uppercase instead of lowercase, and start with CELERY_, # so for example the task_always_eager setting becomes CELERY_TASK_ALWAYS_EAGER, app.config_from_object("django.conf:settings", namespace="CELERY") diff --git a/notification/__init__.py b/notification/__init__.py index 56f53fe2..37d3dd7e 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,26 +1,30 @@ class NotifyMixin: def get_notification_recipients(self): """ - Should Query the users based on their notification settings + Should Query the users based on their notification settings and the entity. Raises: NotImplementedError: It should be implemented by the subclass """ - raise NotImplementedError(f"Subclasses of {NotifyMixin.__name__} must implement {self.get_notification_recipients.__name__}") - + raise NotImplementedError( + f"Subclasses of {NotifyMixin.__name__} must implement {self.get_notification_recipients.__name__}" + ) + def make_notification(self): """ - Creates a notifications for the reciepients based on + Creates a notifications for the reciepients based on the business logic of the entity. Raises: NotImplementedError: It should be implemented by the subclass """ - raise NotImplementedError(f"Subclasses of {NotifyMixin.__name__} must implement {self.make_notification.__name__}") - + raise NotImplementedError( + f"Subclasses of {NotifyMixin.__name__} must implement {self.make_notification.__name__}" + ) + def get_absolute_url(self): """ Returns the absolute url of the entity. """ - return None \ No newline at end of file + return None diff --git a/notification/tasks.py b/notification/tasks.py index 560d54da..c7b3cea4 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -19,6 +19,7 @@ NotificationStyle.once_per_month: timedelta(days=33), } + @shared_task def create_notifications_for_entities(): """ @@ -27,8 +28,11 @@ def create_notifications_for_entities(): """ now = timezone.now() + # Get all the users that are local custodians of a contract + pass + @shared_task def send_notifications_for_user_by_time(user_id, time): """ From 36712e3a271eba56b978e3f34926a2fe444bc05a Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 23 Oct 2023 17:11:28 +0200 Subject: [PATCH 05/38] start the dataset notification logic --- core/models/dataset.py | 14 +++++++++++--- notification/__init__.py | 16 ++++++++++++---- notification/tasks.py | 4 ---- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/core/models/dataset.py b/core/models/dataset.py index 629c21c1..f17b2516 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -1,19 +1,20 @@ import uuid from django.conf import settings +from django.contrib.auth import get_user_model from django.db import models from django.urls import reverse from django.utils.module_loading import import_string - from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase + from core import constants from core.permissions.mapping import PERMISSION_MAPPING - +from notification import NotifyMixin from .utils import CoreTrackedModel, TextFieldWithInputWidget from .partner import HomeOrganisation -class Dataset(CoreTrackedModel): +class Dataset(CoreTrackedModel, NotifyMixin): class Meta: app_label = "core" get_latest_by = "added" @@ -227,6 +228,13 @@ def publish(self, save=True): for data_declaration in self.data_declarations.all(): data_declaration.publish_subentities() + def get_notification_recipients(): + """ + Get distinct users that are local custodian of a dataset. + """ + + return get_user_model().objects.filter(datasets__isnull=False).distinct() + # faster lookup for permissions # https://django-guardian.readthedocs.io/en/stable/userguide/performance.html#direct-foreign-keys diff --git a/notification/__init__.py b/notification/__init__.py index 37d3dd7e..777edcd6 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,5 +1,9 @@ +from django.db.models.query import QuerySet + + class NotifyMixin: - def get_notification_recipients(self): + @classmethod + def get_notification_recipients(cls): """ Should Query the users based on their notification settings and the entity. @@ -8,19 +12,23 @@ def get_notification_recipients(self): NotImplementedError: It should be implemented by the subclass """ raise NotImplementedError( - f"Subclasses of {NotifyMixin.__name__} must implement {self.get_notification_recipients.__name__}" + f"Subclasses of {NotifyMixin.__name__} must implement {cls.get_notification_recipients.__name__}" ) - def make_notification(self): + @classmethod + def make_notification(cls, recipients: QuerySet): """ Creates a notifications for the reciepients based on the business logic of the entity. + Params: + recipients: The users to notify + Raises: NotImplementedError: It should be implemented by the subclass """ raise NotImplementedError( - f"Subclasses of {NotifyMixin.__name__} must implement {self.make_notification.__name__}" + f"Subclasses of {NotifyMixin.__name__} must implement {cls.make_notification.__name__}" ) def get_absolute_url(self): diff --git a/notification/tasks.py b/notification/tasks.py index c7b3cea4..dc0151a3 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -28,10 +28,6 @@ def create_notifications_for_entities(): """ now = timezone.now() - # Get all the users that are local custodians of a contract - - pass - @shared_task def send_notifications_for_user_by_time(user_id, time): From 08d63dd574a1a67a46b7f344e0c48dd019537a5f Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 25 Oct 2023 14:33:49 +0200 Subject: [PATCH 06/38] implement the notification mixin for dataset logic --- core/models/dataset.py | 69 +++++++++- notification/__init__.py | 21 ++- notification/models.py | 6 +- notification/tasks.py | 278 ++++++++++++++++++++------------------- 4 files changed, 231 insertions(+), 143 deletions(-) diff --git a/core/models/dataset.py b/core/models/dataset.py index f17b2516..56672996 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -1,15 +1,21 @@ import uuid +import datetime +from datetime import timedelta from django.conf import settings from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType from django.db import models +from django.db.models import Q from django.urls import reverse from django.utils.module_loading import import_string from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from core import constants +from core.models import DataDeclaration from core.permissions.mapping import PERMISSION_MAPPING from notification import NotifyMixin +from notification.models import Notification, NotificationVerb from .utils import CoreTrackedModel, TextFieldWithInputWidget from .partner import HomeOrganisation @@ -228,12 +234,73 @@ def publish(self, save=True): for data_declaration in self.data_declarations.all(): data_declaration.publish_subentities() + @staticmethod def get_notification_recipients(): """ Get distinct users that are local custodian of a dataset. """ - return get_user_model().objects.filter(datasets__isnull=False).distinct() + return ( + get_user_model() + .objects.filter(Q(datasets__isnull=False) | Q(projects__isnull=False)) + .distinct() + ) + + @classmethod + def make_notifications(cls, exec_date: datetime.date): + recipients = cls.get_notification_recipients() + for user in recipients: + notification_setting = user.notification_setting + if not ( + notification_setting.send_email or notification_setting.send_in_app + ): + continue + day_offset = timedelta(days=notification_setting.notification_offset) + + # Considering users that are indirectly responsible for the dataset (through projects) + possible_datasets = set( + list(user.datasets.all()) + + [p.datasets.all() for p in user.projects.all()] + ) + for dataset in possible_datasets: + # Data Declaration (Embargo Date & End of Storage Duration) + for data_declaration in dataset.data_declarations.all(): + if ( + data_declaration.embargo_date + and data_declaration.embargo_date.date() - day_offset + == exec_date + ): + cls.notify(user, data_declaration, NotificationVerb.embargo_end) + if ( + data_declaration.end_of_storage_duration + and data_declaration.end_of_storage_duration.date() - day_offset + == exec_date + ): + cls.notify(user, data_declaration, NotificationVerb.end) + + @staticmethod + def notify( + user: settings.AUTH_USER_MODEL, obj: "DataDeclaration", verb: NotificationVerb + ): + """ + Notifies concerning users about the entity. + """ + offset = user.notification_setting.notification_offset + + if verb == NotificationVerb.embargo_end: + msg = f"Embargo for {obj.dataset.title} is ending in {offset} days." + else: + msg = ( + f"Storage duration for {obj.dataset.title} is ending in {offset} days." + ) + + Notification.objects.create( + recipient=user, + verb=verb, + msg=msg, + content_type=ContentType.objects.get_for_model(obj.dataset), + object_id=obj.dataset.id, + ).save() # faster lookup for permissions diff --git a/notification/__init__.py b/notification/__init__.py index 777edcd6..42265288 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,9 +1,9 @@ -from django.db.models.query import QuerySet +from datetime import date class NotifyMixin: - @classmethod - def get_notification_recipients(cls): + @staticmethod + def get_notification_recipients(): """ Should Query the users based on their notification settings and the entity. @@ -12,17 +12,17 @@ def get_notification_recipients(cls): NotImplementedError: It should be implemented by the subclass """ raise NotImplementedError( - f"Subclasses of {NotifyMixin.__name__} must implement {cls.get_notification_recipients.__name__}" + f"Subclasses of {NotifyMixin.__name__} must implement {NotifyMixin.get_notification_recipients.__name__}" ) @classmethod - def make_notification(cls, recipients: QuerySet): + def make_notifications(cls, exec_date: date): """ Creates a notifications for the reciepients based on the business logic of the entity. Params: - recipients: The users to notify + exec_date: The date of execution Raises: NotImplementedError: It should be implemented by the subclass @@ -31,6 +31,15 @@ def make_notification(cls, recipients: QuerySet): f"Subclasses of {NotifyMixin.__name__} must implement {cls.make_notification.__name__}" ) + @staticmethod + def notify(obj: object, verb): + """ + Notify the user about the entity. + """ + raise NotImplementedError( + f"Subclasses of {NotifyMixin.__name__} must implement {NotifyMixin.notify.__name__}" + ) + def get_absolute_url(self): """ Returns the absolute url of the entity. diff --git a/notification/models.py b/notification/models.py index 80f2431f..710494ec 100644 --- a/notification/models.py +++ b/notification/models.py @@ -36,7 +36,9 @@ class Meta: app_label = "notification" user = models.OneToOneField( - get_user_model(), related_name="notification_setting", on_delete=models.CASCADE + settings.AUTH_USER_MODEL, + related_name="notification_setting", + on_delete=models.CASCADE, ) send_email = models.BooleanField(default=False) send_in_app = models.BooleanField(default=True) @@ -64,7 +66,7 @@ class Meta: app_label = "notification" recipient = models.ForeignKey( - get_user_model(), related_name="notifications", on_delete=models.CASCADE + settings.AUTH_USER_MODEL, related_name="notifications", on_delete=models.CASCADE ) verb = EnumChoiceField(NotificationVerb) sent_in_app = models.BooleanField(default=True) diff --git a/notification/tasks.py b/notification/tasks.py index dc0151a3..c03fd5a1 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime from collections import defaultdict from datetime import timedelta @@ -10,6 +10,7 @@ from core.models import Contract, DataDeclaration, Dataset, Document, Project, User from notification.email_sender import send_the_email from notification.models import Notification, NotificationStyle, NotificationVerb +from notification import NotifyMixin # map each notification style to a delta # delta correspond to the interval + a small delta @@ -21,141 +22,150 @@ @shared_task -def create_notifications_for_entities(): +def create_notifications_for_entities(executation_date: str = None): """ Loops Through all the entitie that implement the Notificaiton Mixin and creates a notification for each one of them according to the logic. - """ - now = timezone.now() - - -@shared_task -def send_notifications_for_user_by_time(user_id, time): - """ - Send a notification report for the current user from the date to the most recent. - """ - # get latest notifications for the user, grouped by verb - notifications = Notification.objects.filter(actor__pk=user_id, time__gte=time) - if not notifications: - return - - # group notifications per verb - notifications_by_verb = defaultdict(list) - for notif in notifications: - notifications_by_verb[notif.verb].append(notif) - - user = User.objects.get(pk=user_id) - context = {"time": time, "user": user, "notifications": dict(notifications_by_verb)} - send_the_email( - settings.EMAIL_DONOTREPLY, - user.email, - "Notifications", - "notification/notification_report", - context, - ) - - -@shared_task -def send_dataset_notification_for_user(user_id, dataset_id, created): - """ - Send the notification that a dataset have been updated. + Params: + executation_date: The date of the execution of the task. FORMAT: YYYY-MM-DD (DEFAULT: Today) """ - dataset = Dataset.objects.get(pk=dataset_id) - user = User.objects.get(pk=user_id) - context = {"user": user, "dataset": dataset, "created": created} - send_the_email( - settings.EMAIL_DONOTREPLY, - user.email, - "Notifications", - "notification/dataset_notification", - context, - ) - - -@shared_task -def send_notifications(period): - """ - Send notifications for each user based on the period selected. - Period must be one of `NotificationStyle` but 'every_time'. - """ - notification_style = NotificationStyle[period] - if notification_style is NotificationStyle.every_time: - raise KeyError("Key not permitted") - # get users with this setting - users = User.objects.filter( - notification_setting__style=notification_style - ).distinct() - # map the setting to a timeperiod - now = timezone.now() - time = now - NOTIFICATION_MAPPING[notification_style] - - # get latest notifications - users = users.filter(notifications__time__gte=time) - for user in users: - send_notifications_for_user_by_time.delay(user.id, time) - return users - - -@shared_task -def data_storage_expiry_notifications(): - now = timezone.now() - - # the user will receive notifications on two consecutive days prior to storage end date - window_2_start = now + datetime.timedelta(days=1) - window_2_end = now + datetime.timedelta(days=2) - - # the user will receive notifications on two consecutive days, two months prior to storage end date - window_60_start = now + datetime.timedelta(days=59) - window_60_end = now + datetime.timedelta(days=60) - - data_declarations = DataDeclaration.objects.filter( - Q( - end_of_storage_duration__gte=window_60_start, - end_of_storage_duration__lte=window_60_end, - ) - | Q( - end_of_storage_duration__gte=window_2_start, - end_of_storage_duration__lte=window_2_end, - ) - ).order_by("end_of_storage_duration") - - for ddec in data_declarations: - for custodian in ddec.dataset.local_custodians.all(): - Notification.objects.create( - actor=custodian, - verb=NotificationVerb.data_storage_expiry, - content_object=ddec, - ) - - -@shared_task -def document_expiry_notifications(): - now = timezone.now() - - # the user will receive notifications on two consecutive days prior to storage end date - window_2_start = now + datetime.timedelta(days=1) - window_2_end = now + datetime.timedelta(days=2) - - # the user will receive notifications on two consecutive days, two months prior to storage end date - window_60_start = now + datetime.timedelta(days=59) - window_60_end = now + datetime.timedelta(days=60) - - documents = Document.objects.filter( - Q(expiry_date__gte=window_60_start, expiry_date__lte=window_60_end) - | Q(expiry_date__gte=window_2_start, expiry_date__lte=window_2_end) - ).order_by("expiry_date") - - for document in documents: - print(document.content_type) - if str(document.content_type) == "project": - obj = Project.objects.get(pk=document.object_id) - if str(document.content_type) == "contract": - obj = Contract.objects.get(pk=document.object_id) - if obj: - for custodian in obj.local_custodians.all(): - Notification.objects.create( - actor=custodian, - verb=NotificationVerb.document_expiry, - content_object=obj, - ) + if not executation_date: + exec_date = datetime.now().date() + else: + exec_date = datetime.strptime(executation_date, "%Y-%m-%d").date() + + for cls in NotifyMixin.__subclasses__(): + cls.make_notifications(exec_date) + + +# @shared_task +# def send_notifications_for_user_by_time(user_id, time): +# """ +# Send a notification report for the current user from the date to the most recent. +# """ +# # get latest notifications for the user, grouped by verb +# notifications = Notification.objects.filter(actor__pk=user_id, time__gte=time) + +# if not notifications: +# return + +# # group notifications per verb +# notifications_by_verb = defaultdict(list) +# for notif in notifications: +# notifications_by_verb[notif.verb].append(notif) + +# user = User.objects.get(pk=user_id) +# context = {"time": time, "user": user, "notifications": dict(notifications_by_verb)} +# send_the_email( +# settings.EMAIL_DONOTREPLY, +# user.email, +# "Notifications", +# "notification/notification_report", +# context, +# ) + + +# @shared_task +# def send_dataset_notification_for_user(user_id, dataset_id, created): +# """ +# Send the notification that a dataset have been updated. +# """ +# dataset = Dataset.objects.get(pk=dataset_id) +# user = User.objects.get(pk=user_id) +# context = {"user": user, "dataset": dataset, "created": created} +# send_the_email( +# settings.EMAIL_DONOTREPLY, +# user.email, +# "Notifications", +# "notification/dataset_notification", +# context, +# ) + + +# @shared_task +# def send_notifications(period): +# """ +# Send notifications for each user based on the period selected. +# Period must be one of `NotificationStyle` but 'every_time'. +# """ +# notification_style = NotificationStyle[period] +# if notification_style is NotificationStyle.every_time: +# raise KeyError("Key not permitted") +# # get users with this setting +# users = User.objects.filter( +# notification_setting__style=notification_style +# ).distinct() +# # map the setting to a timeperiod +# now = timezone.now() +# time = now - NOTIFICATION_MAPPING[notification_style] + +# # get latest notifications +# users = users.filter(notifications__time__gte=time) +# for user in users: +# send_notifications_for_user_by_time.delay(user.id, time) +# return users + + +# @shared_task +# def data_storage_expiry_notifications(): +# now = timezone.now() + +# # the user will receive notifications on two consecutive days prior to storage end date +# window_2_start = now + datetime.timedelta(days=1) +# window_2_end = now + datetime.timedelta(days=2) + +# # the user will receive notifications on two consecutive days, two months prior to storage end date +# window_60_start = now + datetime.timedelta(days=59) +# window_60_end = now + datetime.timedelta(days=60) + +# data_declarations = DataDeclaration.objects.filter( +# Q( +# end_of_storage_duration__gte=window_60_start, +# end_of_storage_duration__lte=window_60_end, +# ) +# | Q( +# end_of_storage_duration__gte=window_2_start, +# end_of_storage_duration__lte=window_2_end, +# ) +# ).order_by("end_of_storage_duration") + +# for ddec in data_declarations: +# for custodian in ddec.dataset.local_custodians.all(): +# Notification.objects.create( +# actor=custodian, +# verb=NotificationVerb.data_storage_expiry, +# content_object=ddec, +# ) + + +# @shared_task +# def document_expiry_notifications(): +# now = timezone.now() + +# # the user will receive notifications on two consecutive days prior to storage end date +# window_2_start = now + datetime.timedelta(days=1) +# window_2_end = now + datetime.timedelta(days=2) + +# # the user will receive notifications on two consecutive days, two months prior to storage end date +# window_60_start = now + datetime.timedelta(days=59) +# window_60_end = now + datetime.timedelta(days=60) + +# documents = Document.objects.filter( +# Q(expiry_date__gte=window_60_start, expiry_date__lte=window_60_end) +# | Q(expiry_date__gte=window_2_start, expiry_date__lte=window_2_end) +# ).order_by("expiry_date") + +# for document in documents: +# print(document.content_type) +# if str(document.content_type) == "project": +# obj = Project.objects.get(pk=document.object_id) +# if str(document.content_type) == "contract": +# obj = Contract.objects.get(pk=document.object_id) +# if obj: +# for custodian in obj.local_custodians.all(): +# Notification.objects.create( +# actor=custodian, +# verb=NotificationVerb.document_expiry, +# content_object=obj, +# ) From 43cb7674ffa122299a5d277f475aba21b7ce2906 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 18 Oct 2023 16:24:01 +0200 Subject: [PATCH 07/38] fix importing local settings for docker settings and refine compose file --- docker-compose.yaml | 2 -- elixir_daisy/settings_compose.py | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 16395d7c..e8653ec3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,8 +12,6 @@ services: - statics:/static - solrdata:/solr - .:/code - ports: - - "5000:5000" depends_on: - db - solr diff --git a/elixir_daisy/settings_compose.py b/elixir_daisy/settings_compose.py index d8e26819..92aa66b9 100644 --- a/elixir_daisy/settings_compose.py +++ b/elixir_daisy/settings_compose.py @@ -71,6 +71,10 @@ TESTING = os.environ.get("TEST", False) # import also local settings +try: + from .settings_local import * +except ImportError as e: + pass try: if TESTING: from .settings_ci import * From 0acdd123204df5773db8525d460aac0c40810e32 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 19 Oct 2023 16:44:20 +0200 Subject: [PATCH 08/38] refine celery config and create the task definition --- elixir_daisy/celery_app.py | 10 ++- elixir_daisy/settings.py | 9 ++- notification/tasks.py | 161 +++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 notification/tasks.py diff --git a/elixir_daisy/celery_app.py b/elixir_daisy/celery_app.py index daa6ffd6..73c1e2cf 100644 --- a/elixir_daisy/celery_app.py +++ b/elixir_daisy/celery_app.py @@ -2,10 +2,16 @@ from celery import Celery -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "elixir_daisy.settings") - app = Celery("daisy") +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +# The uppercase name-space means that all Celery configuration options must +# be specified in uppercase instead of lowercase, and start with CELERY_, +# so for example the task_always_eager setting becomes CELERY_TASK_ALWAYS_EAGER, + app.config_from_object("django.conf:settings", namespace="CELERY") # Load task modules from all registered Django app configs. diff --git a/elixir_daisy/settings.py b/elixir_daisy/settings.py index e8cf869e..faf6cf34 100644 --- a/elixir_daisy/settings.py +++ b/elixir_daisy/settings.py @@ -144,16 +144,17 @@ # https://docs.djangoproject.com/en/2.0/topics/i18n/ LANGUAGE_CODE = "en-us" - TIME_ZONE = "Europe/Luxembourg" TZINFO = pytz.timezone(TIME_ZONE) - USE_I18N = True - USE_L10N = True - USE_TZ = True +# Celery configs + +CELERY_BROKER_URL = "amqp://guest:guest@localhost:5672//" +CELERY_TIMEZONE = "Europe/Luxembourg" + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ STATIC_URL = "/static/" diff --git a/notification/tasks.py b/notification/tasks.py new file mode 100644 index 00000000..560d54da --- /dev/null +++ b/notification/tasks.py @@ -0,0 +1,161 @@ +import datetime +from collections import defaultdict +from datetime import timedelta + +from celery import shared_task +from django.conf import settings +from django.utils import timezone +from django.db.models import Q + +from core.models import Contract, DataDeclaration, Dataset, Document, Project, User +from notification.email_sender import send_the_email +from notification.models import Notification, NotificationStyle, NotificationVerb + +# map each notification style to a delta +# delta correspond to the interval + a small delta +NOTIFICATION_MAPPING = { + NotificationStyle.once_per_day: timedelta(days=1, hours=8), + NotificationStyle.once_per_week: timedelta(days=7, hours=16), + NotificationStyle.once_per_month: timedelta(days=33), +} + +@shared_task +def create_notifications_for_entities(): + """ + Loops Through all the entitie that implement the Notificaiton Mixin + and creates a notification for each one of them according to the logic. + """ + now = timezone.now() + + pass + +@shared_task +def send_notifications_for_user_by_time(user_id, time): + """ + Send a notification report for the current user from the date to the most recent. + """ + # get latest notifications for the user, grouped by verb + notifications = Notification.objects.filter(actor__pk=user_id, time__gte=time) + + if not notifications: + return + + # group notifications per verb + notifications_by_verb = defaultdict(list) + for notif in notifications: + notifications_by_verb[notif.verb].append(notif) + + user = User.objects.get(pk=user_id) + context = {"time": time, "user": user, "notifications": dict(notifications_by_verb)} + send_the_email( + settings.EMAIL_DONOTREPLY, + user.email, + "Notifications", + "notification/notification_report", + context, + ) + + +@shared_task +def send_dataset_notification_for_user(user_id, dataset_id, created): + """ + Send the notification that a dataset have been updated. + """ + dataset = Dataset.objects.get(pk=dataset_id) + user = User.objects.get(pk=user_id) + context = {"user": user, "dataset": dataset, "created": created} + send_the_email( + settings.EMAIL_DONOTREPLY, + user.email, + "Notifications", + "notification/dataset_notification", + context, + ) + + +@shared_task +def send_notifications(period): + """ + Send notifications for each user based on the period selected. + Period must be one of `NotificationStyle` but 'every_time'. + """ + notification_style = NotificationStyle[period] + if notification_style is NotificationStyle.every_time: + raise KeyError("Key not permitted") + # get users with this setting + users = User.objects.filter( + notification_setting__style=notification_style + ).distinct() + # map the setting to a timeperiod + now = timezone.now() + time = now - NOTIFICATION_MAPPING[notification_style] + + # get latest notifications + users = users.filter(notifications__time__gte=time) + for user in users: + send_notifications_for_user_by_time.delay(user.id, time) + return users + + +@shared_task +def data_storage_expiry_notifications(): + now = timezone.now() + + # the user will receive notifications on two consecutive days prior to storage end date + window_2_start = now + datetime.timedelta(days=1) + window_2_end = now + datetime.timedelta(days=2) + + # the user will receive notifications on two consecutive days, two months prior to storage end date + window_60_start = now + datetime.timedelta(days=59) + window_60_end = now + datetime.timedelta(days=60) + + data_declarations = DataDeclaration.objects.filter( + Q( + end_of_storage_duration__gte=window_60_start, + end_of_storage_duration__lte=window_60_end, + ) + | Q( + end_of_storage_duration__gte=window_2_start, + end_of_storage_duration__lte=window_2_end, + ) + ).order_by("end_of_storage_duration") + + for ddec in data_declarations: + for custodian in ddec.dataset.local_custodians.all(): + Notification.objects.create( + actor=custodian, + verb=NotificationVerb.data_storage_expiry, + content_object=ddec, + ) + + +@shared_task +def document_expiry_notifications(): + now = timezone.now() + + # the user will receive notifications on two consecutive days prior to storage end date + window_2_start = now + datetime.timedelta(days=1) + window_2_end = now + datetime.timedelta(days=2) + + # the user will receive notifications on two consecutive days, two months prior to storage end date + window_60_start = now + datetime.timedelta(days=59) + window_60_end = now + datetime.timedelta(days=60) + + documents = Document.objects.filter( + Q(expiry_date__gte=window_60_start, expiry_date__lte=window_60_end) + | Q(expiry_date__gte=window_2_start, expiry_date__lte=window_2_end) + ).order_by("expiry_date") + + for document in documents: + print(document.content_type) + if str(document.content_type) == "project": + obj = Project.objects.get(pk=document.object_id) + if str(document.content_type) == "contract": + obj = Contract.objects.get(pk=document.object_id) + if obj: + for custodian in obj.local_custodians.all(): + Notification.objects.create( + actor=custodian, + verb=NotificationVerb.document_expiry, + content_object=obj, + ) From 0e574800e8d5e469264a88147936b7c8ef146c6b Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Fri, 20 Oct 2023 11:58:24 +0200 Subject: [PATCH 09/38] add NotifyMixin --- notification/__init__.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/notification/__init__.py b/notification/__init__.py index e69de29b..56f53fe2 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -0,0 +1,26 @@ +class NotifyMixin: + def get_notification_recipients(self): + """ + Should Query the users based on their notification settings + and the entity. + + Raises: + NotImplementedError: It should be implemented by the subclass + """ + raise NotImplementedError(f"Subclasses of {NotifyMixin.__name__} must implement {self.get_notification_recipients.__name__}") + + def make_notification(self): + """ + Creates a notifications for the reciepients based on + the business logic of the entity. + + Raises: + NotImplementedError: It should be implemented by the subclass + """ + raise NotImplementedError(f"Subclasses of {NotifyMixin.__name__} must implement {self.make_notification.__name__}") + + def get_absolute_url(self): + """ + Returns the absolute url of the entity. + """ + return None \ No newline at end of file From 2ba2e30adb4d843ae431e4d11fce69b7a50b8a4a Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 23 Oct 2023 14:02:30 +0200 Subject: [PATCH 10/38] apply black linter --- elixir_daisy/celery_app.py | 2 +- notification/__init__.py | 18 +++++++++++------- notification/tasks.py | 4 ++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/elixir_daisy/celery_app.py b/elixir_daisy/celery_app.py index 73c1e2cf..790e64f5 100644 --- a/elixir_daisy/celery_app.py +++ b/elixir_daisy/celery_app.py @@ -9,7 +9,7 @@ # - namespace='CELERY' means all celery-related configuration keys # should have a `CELERY_` prefix. # The uppercase name-space means that all Celery configuration options must -# be specified in uppercase instead of lowercase, and start with CELERY_, +# be specified in uppercase instead of lowercase, and start with CELERY_, # so for example the task_always_eager setting becomes CELERY_TASK_ALWAYS_EAGER, app.config_from_object("django.conf:settings", namespace="CELERY") diff --git a/notification/__init__.py b/notification/__init__.py index 56f53fe2..37d3dd7e 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,26 +1,30 @@ class NotifyMixin: def get_notification_recipients(self): """ - Should Query the users based on their notification settings + Should Query the users based on their notification settings and the entity. Raises: NotImplementedError: It should be implemented by the subclass """ - raise NotImplementedError(f"Subclasses of {NotifyMixin.__name__} must implement {self.get_notification_recipients.__name__}") - + raise NotImplementedError( + f"Subclasses of {NotifyMixin.__name__} must implement {self.get_notification_recipients.__name__}" + ) + def make_notification(self): """ - Creates a notifications for the reciepients based on + Creates a notifications for the reciepients based on the business logic of the entity. Raises: NotImplementedError: It should be implemented by the subclass """ - raise NotImplementedError(f"Subclasses of {NotifyMixin.__name__} must implement {self.make_notification.__name__}") - + raise NotImplementedError( + f"Subclasses of {NotifyMixin.__name__} must implement {self.make_notification.__name__}" + ) + def get_absolute_url(self): """ Returns the absolute url of the entity. """ - return None \ No newline at end of file + return None diff --git a/notification/tasks.py b/notification/tasks.py index 560d54da..c7b3cea4 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -19,6 +19,7 @@ NotificationStyle.once_per_month: timedelta(days=33), } + @shared_task def create_notifications_for_entities(): """ @@ -27,8 +28,11 @@ def create_notifications_for_entities(): """ now = timezone.now() + # Get all the users that are local custodians of a contract + pass + @shared_task def send_notifications_for_user_by_time(user_id, time): """ From 132d81375f32ee49746f6f3297d122da84aee7ec Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 23 Oct 2023 17:11:28 +0200 Subject: [PATCH 11/38] start the dataset notification logic --- core/models/dataset.py | 14 +++++++++++--- notification/__init__.py | 16 ++++++++++++---- notification/tasks.py | 4 ---- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/core/models/dataset.py b/core/models/dataset.py index 629c21c1..f17b2516 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -1,19 +1,20 @@ import uuid from django.conf import settings +from django.contrib.auth import get_user_model from django.db import models from django.urls import reverse from django.utils.module_loading import import_string - from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase + from core import constants from core.permissions.mapping import PERMISSION_MAPPING - +from notification import NotifyMixin from .utils import CoreTrackedModel, TextFieldWithInputWidget from .partner import HomeOrganisation -class Dataset(CoreTrackedModel): +class Dataset(CoreTrackedModel, NotifyMixin): class Meta: app_label = "core" get_latest_by = "added" @@ -227,6 +228,13 @@ def publish(self, save=True): for data_declaration in self.data_declarations.all(): data_declaration.publish_subentities() + def get_notification_recipients(): + """ + Get distinct users that are local custodian of a dataset. + """ + + return get_user_model().objects.filter(datasets__isnull=False).distinct() + # faster lookup for permissions # https://django-guardian.readthedocs.io/en/stable/userguide/performance.html#direct-foreign-keys diff --git a/notification/__init__.py b/notification/__init__.py index 37d3dd7e..777edcd6 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,5 +1,9 @@ +from django.db.models.query import QuerySet + + class NotifyMixin: - def get_notification_recipients(self): + @classmethod + def get_notification_recipients(cls): """ Should Query the users based on their notification settings and the entity. @@ -8,19 +12,23 @@ def get_notification_recipients(self): NotImplementedError: It should be implemented by the subclass """ raise NotImplementedError( - f"Subclasses of {NotifyMixin.__name__} must implement {self.get_notification_recipients.__name__}" + f"Subclasses of {NotifyMixin.__name__} must implement {cls.get_notification_recipients.__name__}" ) - def make_notification(self): + @classmethod + def make_notification(cls, recipients: QuerySet): """ Creates a notifications for the reciepients based on the business logic of the entity. + Params: + recipients: The users to notify + Raises: NotImplementedError: It should be implemented by the subclass """ raise NotImplementedError( - f"Subclasses of {NotifyMixin.__name__} must implement {self.make_notification.__name__}" + f"Subclasses of {NotifyMixin.__name__} must implement {cls.make_notification.__name__}" ) def get_absolute_url(self): diff --git a/notification/tasks.py b/notification/tasks.py index c7b3cea4..dc0151a3 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -28,10 +28,6 @@ def create_notifications_for_entities(): """ now = timezone.now() - # Get all the users that are local custodians of a contract - - pass - @shared_task def send_notifications_for_user_by_time(user_id, time): From ed6f369281d1523737df435dc233a0f7c50cf7ac Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 25 Oct 2023 14:33:49 +0200 Subject: [PATCH 12/38] implement the notification mixin for dataset logic --- core/models/dataset.py | 69 +++++++++- notification/__init__.py | 21 ++- notification/models.py | 6 +- notification/tasks.py | 278 ++++++++++++++++++++------------------- 4 files changed, 231 insertions(+), 143 deletions(-) diff --git a/core/models/dataset.py b/core/models/dataset.py index f17b2516..56672996 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -1,15 +1,21 @@ import uuid +import datetime +from datetime import timedelta from django.conf import settings from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType from django.db import models +from django.db.models import Q from django.urls import reverse from django.utils.module_loading import import_string from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from core import constants +from core.models import DataDeclaration from core.permissions.mapping import PERMISSION_MAPPING from notification import NotifyMixin +from notification.models import Notification, NotificationVerb from .utils import CoreTrackedModel, TextFieldWithInputWidget from .partner import HomeOrganisation @@ -228,12 +234,73 @@ def publish(self, save=True): for data_declaration in self.data_declarations.all(): data_declaration.publish_subentities() + @staticmethod def get_notification_recipients(): """ Get distinct users that are local custodian of a dataset. """ - return get_user_model().objects.filter(datasets__isnull=False).distinct() + return ( + get_user_model() + .objects.filter(Q(datasets__isnull=False) | Q(projects__isnull=False)) + .distinct() + ) + + @classmethod + def make_notifications(cls, exec_date: datetime.date): + recipients = cls.get_notification_recipients() + for user in recipients: + notification_setting = user.notification_setting + if not ( + notification_setting.send_email or notification_setting.send_in_app + ): + continue + day_offset = timedelta(days=notification_setting.notification_offset) + + # Considering users that are indirectly responsible for the dataset (through projects) + possible_datasets = set( + list(user.datasets.all()) + + [p.datasets.all() for p in user.projects.all()] + ) + for dataset in possible_datasets: + # Data Declaration (Embargo Date & End of Storage Duration) + for data_declaration in dataset.data_declarations.all(): + if ( + data_declaration.embargo_date + and data_declaration.embargo_date.date() - day_offset + == exec_date + ): + cls.notify(user, data_declaration, NotificationVerb.embargo_end) + if ( + data_declaration.end_of_storage_duration + and data_declaration.end_of_storage_duration.date() - day_offset + == exec_date + ): + cls.notify(user, data_declaration, NotificationVerb.end) + + @staticmethod + def notify( + user: settings.AUTH_USER_MODEL, obj: "DataDeclaration", verb: NotificationVerb + ): + """ + Notifies concerning users about the entity. + """ + offset = user.notification_setting.notification_offset + + if verb == NotificationVerb.embargo_end: + msg = f"Embargo for {obj.dataset.title} is ending in {offset} days." + else: + msg = ( + f"Storage duration for {obj.dataset.title} is ending in {offset} days." + ) + + Notification.objects.create( + recipient=user, + verb=verb, + msg=msg, + content_type=ContentType.objects.get_for_model(obj.dataset), + object_id=obj.dataset.id, + ).save() # faster lookup for permissions diff --git a/notification/__init__.py b/notification/__init__.py index 777edcd6..42265288 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,9 +1,9 @@ -from django.db.models.query import QuerySet +from datetime import date class NotifyMixin: - @classmethod - def get_notification_recipients(cls): + @staticmethod + def get_notification_recipients(): """ Should Query the users based on their notification settings and the entity. @@ -12,17 +12,17 @@ def get_notification_recipients(cls): NotImplementedError: It should be implemented by the subclass """ raise NotImplementedError( - f"Subclasses of {NotifyMixin.__name__} must implement {cls.get_notification_recipients.__name__}" + f"Subclasses of {NotifyMixin.__name__} must implement {NotifyMixin.get_notification_recipients.__name__}" ) @classmethod - def make_notification(cls, recipients: QuerySet): + def make_notifications(cls, exec_date: date): """ Creates a notifications for the reciepients based on the business logic of the entity. Params: - recipients: The users to notify + exec_date: The date of execution Raises: NotImplementedError: It should be implemented by the subclass @@ -31,6 +31,15 @@ def make_notification(cls, recipients: QuerySet): f"Subclasses of {NotifyMixin.__name__} must implement {cls.make_notification.__name__}" ) + @staticmethod + def notify(obj: object, verb): + """ + Notify the user about the entity. + """ + raise NotImplementedError( + f"Subclasses of {NotifyMixin.__name__} must implement {NotifyMixin.notify.__name__}" + ) + def get_absolute_url(self): """ Returns the absolute url of the entity. diff --git a/notification/models.py b/notification/models.py index 274f18b8..00d6b37d 100644 --- a/notification/models.py +++ b/notification/models.py @@ -36,7 +36,9 @@ class Meta: app_label = "notification" user = models.OneToOneField( - get_user_model(), related_name="notification_setting", on_delete=models.CASCADE + settings.AUTH_USER_MODEL, + related_name="notification_setting", + on_delete=models.CASCADE, ) send_email = models.BooleanField( default=False, @@ -74,7 +76,7 @@ class Meta: app_label = "notification" recipient = models.ForeignKey( - get_user_model(), related_name="notifications", on_delete=models.CASCADE + settings.AUTH_USER_MODEL, related_name="notifications", on_delete=models.CASCADE ) verb = EnumChoiceField(NotificationVerb) on = models.DateTimeField(null=True, blank=True, default=None) diff --git a/notification/tasks.py b/notification/tasks.py index dc0151a3..c03fd5a1 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime from collections import defaultdict from datetime import timedelta @@ -10,6 +10,7 @@ from core.models import Contract, DataDeclaration, Dataset, Document, Project, User from notification.email_sender import send_the_email from notification.models import Notification, NotificationStyle, NotificationVerb +from notification import NotifyMixin # map each notification style to a delta # delta correspond to the interval + a small delta @@ -21,141 +22,150 @@ @shared_task -def create_notifications_for_entities(): +def create_notifications_for_entities(executation_date: str = None): """ Loops Through all the entitie that implement the Notificaiton Mixin and creates a notification for each one of them according to the logic. - """ - now = timezone.now() - - -@shared_task -def send_notifications_for_user_by_time(user_id, time): - """ - Send a notification report for the current user from the date to the most recent. - """ - # get latest notifications for the user, grouped by verb - notifications = Notification.objects.filter(actor__pk=user_id, time__gte=time) - if not notifications: - return - - # group notifications per verb - notifications_by_verb = defaultdict(list) - for notif in notifications: - notifications_by_verb[notif.verb].append(notif) - - user = User.objects.get(pk=user_id) - context = {"time": time, "user": user, "notifications": dict(notifications_by_verb)} - send_the_email( - settings.EMAIL_DONOTREPLY, - user.email, - "Notifications", - "notification/notification_report", - context, - ) - - -@shared_task -def send_dataset_notification_for_user(user_id, dataset_id, created): - """ - Send the notification that a dataset have been updated. + Params: + executation_date: The date of the execution of the task. FORMAT: YYYY-MM-DD (DEFAULT: Today) """ - dataset = Dataset.objects.get(pk=dataset_id) - user = User.objects.get(pk=user_id) - context = {"user": user, "dataset": dataset, "created": created} - send_the_email( - settings.EMAIL_DONOTREPLY, - user.email, - "Notifications", - "notification/dataset_notification", - context, - ) - - -@shared_task -def send_notifications(period): - """ - Send notifications for each user based on the period selected. - Period must be one of `NotificationStyle` but 'every_time'. - """ - notification_style = NotificationStyle[period] - if notification_style is NotificationStyle.every_time: - raise KeyError("Key not permitted") - # get users with this setting - users = User.objects.filter( - notification_setting__style=notification_style - ).distinct() - # map the setting to a timeperiod - now = timezone.now() - time = now - NOTIFICATION_MAPPING[notification_style] - - # get latest notifications - users = users.filter(notifications__time__gte=time) - for user in users: - send_notifications_for_user_by_time.delay(user.id, time) - return users - - -@shared_task -def data_storage_expiry_notifications(): - now = timezone.now() - - # the user will receive notifications on two consecutive days prior to storage end date - window_2_start = now + datetime.timedelta(days=1) - window_2_end = now + datetime.timedelta(days=2) - - # the user will receive notifications on two consecutive days, two months prior to storage end date - window_60_start = now + datetime.timedelta(days=59) - window_60_end = now + datetime.timedelta(days=60) - - data_declarations = DataDeclaration.objects.filter( - Q( - end_of_storage_duration__gte=window_60_start, - end_of_storage_duration__lte=window_60_end, - ) - | Q( - end_of_storage_duration__gte=window_2_start, - end_of_storage_duration__lte=window_2_end, - ) - ).order_by("end_of_storage_duration") - - for ddec in data_declarations: - for custodian in ddec.dataset.local_custodians.all(): - Notification.objects.create( - actor=custodian, - verb=NotificationVerb.data_storage_expiry, - content_object=ddec, - ) - - -@shared_task -def document_expiry_notifications(): - now = timezone.now() - - # the user will receive notifications on two consecutive days prior to storage end date - window_2_start = now + datetime.timedelta(days=1) - window_2_end = now + datetime.timedelta(days=2) - - # the user will receive notifications on two consecutive days, two months prior to storage end date - window_60_start = now + datetime.timedelta(days=59) - window_60_end = now + datetime.timedelta(days=60) - - documents = Document.objects.filter( - Q(expiry_date__gte=window_60_start, expiry_date__lte=window_60_end) - | Q(expiry_date__gte=window_2_start, expiry_date__lte=window_2_end) - ).order_by("expiry_date") - - for document in documents: - print(document.content_type) - if str(document.content_type) == "project": - obj = Project.objects.get(pk=document.object_id) - if str(document.content_type) == "contract": - obj = Contract.objects.get(pk=document.object_id) - if obj: - for custodian in obj.local_custodians.all(): - Notification.objects.create( - actor=custodian, - verb=NotificationVerb.document_expiry, - content_object=obj, - ) + if not executation_date: + exec_date = datetime.now().date() + else: + exec_date = datetime.strptime(executation_date, "%Y-%m-%d").date() + + for cls in NotifyMixin.__subclasses__(): + cls.make_notifications(exec_date) + + +# @shared_task +# def send_notifications_for_user_by_time(user_id, time): +# """ +# Send a notification report for the current user from the date to the most recent. +# """ +# # get latest notifications for the user, grouped by verb +# notifications = Notification.objects.filter(actor__pk=user_id, time__gte=time) + +# if not notifications: +# return + +# # group notifications per verb +# notifications_by_verb = defaultdict(list) +# for notif in notifications: +# notifications_by_verb[notif.verb].append(notif) + +# user = User.objects.get(pk=user_id) +# context = {"time": time, "user": user, "notifications": dict(notifications_by_verb)} +# send_the_email( +# settings.EMAIL_DONOTREPLY, +# user.email, +# "Notifications", +# "notification/notification_report", +# context, +# ) + + +# @shared_task +# def send_dataset_notification_for_user(user_id, dataset_id, created): +# """ +# Send the notification that a dataset have been updated. +# """ +# dataset = Dataset.objects.get(pk=dataset_id) +# user = User.objects.get(pk=user_id) +# context = {"user": user, "dataset": dataset, "created": created} +# send_the_email( +# settings.EMAIL_DONOTREPLY, +# user.email, +# "Notifications", +# "notification/dataset_notification", +# context, +# ) + + +# @shared_task +# def send_notifications(period): +# """ +# Send notifications for each user based on the period selected. +# Period must be one of `NotificationStyle` but 'every_time'. +# """ +# notification_style = NotificationStyle[period] +# if notification_style is NotificationStyle.every_time: +# raise KeyError("Key not permitted") +# # get users with this setting +# users = User.objects.filter( +# notification_setting__style=notification_style +# ).distinct() +# # map the setting to a timeperiod +# now = timezone.now() +# time = now - NOTIFICATION_MAPPING[notification_style] + +# # get latest notifications +# users = users.filter(notifications__time__gte=time) +# for user in users: +# send_notifications_for_user_by_time.delay(user.id, time) +# return users + + +# @shared_task +# def data_storage_expiry_notifications(): +# now = timezone.now() + +# # the user will receive notifications on two consecutive days prior to storage end date +# window_2_start = now + datetime.timedelta(days=1) +# window_2_end = now + datetime.timedelta(days=2) + +# # the user will receive notifications on two consecutive days, two months prior to storage end date +# window_60_start = now + datetime.timedelta(days=59) +# window_60_end = now + datetime.timedelta(days=60) + +# data_declarations = DataDeclaration.objects.filter( +# Q( +# end_of_storage_duration__gte=window_60_start, +# end_of_storage_duration__lte=window_60_end, +# ) +# | Q( +# end_of_storage_duration__gte=window_2_start, +# end_of_storage_duration__lte=window_2_end, +# ) +# ).order_by("end_of_storage_duration") + +# for ddec in data_declarations: +# for custodian in ddec.dataset.local_custodians.all(): +# Notification.objects.create( +# actor=custodian, +# verb=NotificationVerb.data_storage_expiry, +# content_object=ddec, +# ) + + +# @shared_task +# def document_expiry_notifications(): +# now = timezone.now() + +# # the user will receive notifications on two consecutive days prior to storage end date +# window_2_start = now + datetime.timedelta(days=1) +# window_2_end = now + datetime.timedelta(days=2) + +# # the user will receive notifications on two consecutive days, two months prior to storage end date +# window_60_start = now + datetime.timedelta(days=59) +# window_60_end = now + datetime.timedelta(days=60) + +# documents = Document.objects.filter( +# Q(expiry_date__gte=window_60_start, expiry_date__lte=window_60_end) +# | Q(expiry_date__gte=window_2_start, expiry_date__lte=window_2_end) +# ).order_by("expiry_date") + +# for document in documents: +# print(document.content_type) +# if str(document.content_type) == "project": +# obj = Project.objects.get(pk=document.object_id) +# if str(document.content_type) == "contract": +# obj = Contract.objects.get(pk=document.object_id) +# if obj: +# for custodian in obj.local_custodians.all(): +# Notification.objects.create( +# actor=custodian, +# verb=NotificationVerb.document_expiry, +# content_object=obj, +# ) From 4b5e0e6e026b62d62b64792ec76e918836d6a656 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 25 Oct 2023 16:44:56 +0200 Subject: [PATCH 13/38] refine the logic on dataset notif --- core/models/dataset.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/core/models/dataset.py b/core/models/dataset.py index 56672996..aee6973c 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -258,22 +258,20 @@ def make_notifications(cls, exec_date: datetime.date): day_offset = timedelta(days=notification_setting.notification_offset) # Considering users that are indirectly responsible for the dataset (through projects) - possible_datasets = set( - list(user.datasets.all()) - + [p.datasets.all() for p in user.projects.all()] - ) + possible_datasets = set(user.datasets.all()) + for project in user.projects.all(): + possible_datasets.update(list(project.datasets.all())) for dataset in possible_datasets: # Data Declaration (Embargo Date & End of Storage Duration) for data_declaration in dataset.data_declarations.all(): if ( data_declaration.embargo_date - and data_declaration.embargo_date.date() - day_offset - == exec_date + and data_declaration.embargo_date - day_offset == exec_date ): cls.notify(user, data_declaration, NotificationVerb.embargo_end) if ( data_declaration.end_of_storage_duration - and data_declaration.end_of_storage_duration.date() - day_offset + and data_declaration.end_of_storage_duration - day_offset == exec_date ): cls.notify(user, data_declaration, NotificationVerb.end) From e47c3a45708f462c7280d6e08e66a1d9748535d5 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Tue, 31 Oct 2023 14:59:45 +0100 Subject: [PATCH 14/38] add finishing touches for the dataset notification logic --- core/models/dataset.py | 19 +++++++++++++------ notification/__init__.py | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/core/models/dataset.py b/core/models/dataset.py index aee6973c..cdc8c493 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -12,10 +12,10 @@ from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from core import constants -from core.models import DataDeclaration +from core.models import DataDeclaration, User from core.permissions.mapping import PERMISSION_MAPPING from notification import NotifyMixin -from notification.models import Notification, NotificationVerb +from notification.models import Notification, NotificationVerb, NotificationSetting from .utils import CoreTrackedModel, TextFieldWithInputWidget from .partner import HomeOrganisation @@ -250,7 +250,9 @@ def get_notification_recipients(): def make_notifications(cls, exec_date: datetime.date): recipients = cls.get_notification_recipients() for user in recipients: - notification_setting = user.notification_setting + notification_setting: NotificationSetting = ( + user.notification_setting or NotificationSetting() + ) if not ( notification_setting.send_email or notification_setting.send_in_app ): @@ -277,25 +279,30 @@ def make_notifications(cls, exec_date: datetime.date): cls.notify(user, data_declaration, NotificationVerb.end) @staticmethod - def notify( - user: settings.AUTH_USER_MODEL, obj: "DataDeclaration", verb: NotificationVerb - ): + def notify(user: "User", obj: "DataDeclaration", verb: "NotificationVerb"): """ Notifies concerning users about the entity. """ offset = user.notification_setting.notification_offset + dispatch_by_email = user.notification_setting.send_email + dispatch_in_app = user.notification_setting.send_in_app if verb == NotificationVerb.embargo_end: msg = f"Embargo for {obj.dataset.title} is ending in {offset} days." + on = obj.embargo_date else: msg = ( f"Storage duration for {obj.dataset.title} is ending in {offset} days." ) + on = obj.end_of_storage_duration Notification.objects.create( recipient=user, verb=verb, msg=msg, + on=on, + dispatch_by_email=dispatch_by_email, + dispatch_in_app=dispatch_in_app, content_type=ContentType.objects.get_for_model(obj.dataset), object_id=obj.dataset.id, ).save() diff --git a/notification/__init__.py b/notification/__init__.py index 42265288..2ec3f3fa 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,9 +1,17 @@ -from datetime import date +import typing +from typing import List + +if typing.TYPE_CHECKING: + from django.conf import settings + from datetime import date + from notification.models import NotificationVerb + + User = settings.AUTH_USER_MODEL class NotifyMixin: @staticmethod - def get_notification_recipients(): + def get_notification_recipients() -> List["User"]: """ Should Query the users based on their notification settings and the entity. @@ -16,7 +24,7 @@ def get_notification_recipients(): ) @classmethod - def make_notifications(cls, exec_date: date): + def make_notifications(cls, exec_date: "date"): """ Creates a notifications for the reciepients based on the business logic of the entity. @@ -32,7 +40,7 @@ def make_notifications(cls, exec_date: date): ) @staticmethod - def notify(obj: object, verb): + def notify(user: "User", obj: object, verb: "NotificationVerb"): """ Notify the user about the entity. """ @@ -40,7 +48,7 @@ def notify(obj: object, verb): f"Subclasses of {NotifyMixin.__name__} must implement {NotifyMixin.notify.__name__}" ) - def get_absolute_url(self): + def get_absolute_url(self) -> str: """ Returns the absolute url of the entity. """ From 9e72e1c0f317edb5cdea14839bcf8cf4b3fc9577 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Tue, 31 Oct 2023 15:09:17 +0100 Subject: [PATCH 15/38] fix the typing for user without importing directly --- core/models/dataset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/models/dataset.py b/core/models/dataset.py index cdc8c493..ce018b17 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -1,6 +1,7 @@ import uuid import datetime from datetime import timedelta +import typing from django.conf import settings from django.contrib.auth import get_user_model @@ -12,13 +13,16 @@ from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from core import constants -from core.models import DataDeclaration, User +from core.models import DataDeclaration from core.permissions.mapping import PERMISSION_MAPPING from notification import NotifyMixin from notification.models import Notification, NotificationVerb, NotificationSetting from .utils import CoreTrackedModel, TextFieldWithInputWidget from .partner import HomeOrganisation +if typing.TYPE_CHECKING: + User = settings.AUTH_USER_MODEL + class Dataset(CoreTrackedModel, NotifyMixin): class Meta: From 6085112c3d70fc9cf3d384206c0d785ed52004fa Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 2 Nov 2023 13:16:41 +0100 Subject: [PATCH 16/38] add business logic to notify about the accesses --- core/migrations/0035_auto_20231031_1522.py | 27 ++++++++ core/models/access.py | 76 +++++++++++++++++++++- 2 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 core/migrations/0035_auto_20231031_1522.py diff --git a/core/migrations/0035_auto_20231031_1522.py b/core/migrations/0035_auto_20231031_1522.py new file mode 100644 index 00000000..9acb9bb9 --- /dev/null +++ b/core/migrations/0035_auto_20231031_1522.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2.20 on 2023-10-31 14:22 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0034_auto_20230515_1353"), + ] + + operations = [ + migrations.AlterField( + model_name="access", + name="user", + field=models.ForeignKey( + blank=True, + help_text="Use either `contact` or `user`", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="accesses", + to=settings.AUTH_USER_MODEL, + verbose_name="User that has the access", + ), + ), + ] diff --git a/core/models/access.py b/core/models/access.py index 387138f8..99bc6e52 100644 --- a/core/models/access.py +++ b/core/models/access.py @@ -1,8 +1,12 @@ import logging +import typing -from datetime import datetime, date +from datetime import datetime, date, timedelta from typing import List +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.db import models from django.db.models import Q, ObjectDoesNotExist, Count, signals @@ -10,10 +14,15 @@ from enumchoicefield import EnumChoiceField, ChoiceEnum from .utils import CoreModel +from notification import NotifyMixin +from notification.models import NotificationSetting, NotificationVerb, Notification from auditlog.registry import auditlog from auditlog.models import AuditlogHistoryField +if typing.TYPE_CHECKING: + User = settings.AUTH_USER_MODEL + logger = logging.getLogger(__name__) @@ -25,7 +34,7 @@ class StatusChoices(ChoiceEnum): terminated = "Terminated" -class Access(CoreModel): +class Access(CoreModel, NotifyMixin): """ Represents the access given to an internal (LCSB) entity over data storage locations. """ @@ -161,7 +170,7 @@ def clean(self): user = models.ForeignKey( "core.User", - related_name="user", + related_name="accesses", verbose_name="User that has the access", on_delete=models.CASCADE, null=True, @@ -240,5 +249,66 @@ def is_active(self): return True + @staticmethod + def get_notification_recipients(): + """ + Get distinct users that are local custodian of a dataset. + """ + + return ( + get_user_model() + .objects.filter(Q(datasets__isnull=False) | Q(projects__isnull=False)) + .distinct() + ) + + @classmethod + def make_notifications(cls, exec_date: date): + recipients = cls.get_notification_recipients() + for user in recipients: + notification_setting: NotificationSetting = ( + user.notification_setting or NotificationSetting() + ) + if not ( + notification_setting.send_email or notification_setting.send_in_app + ): + continue + day_offset = timedelta(days=notification_setting.notification_offset) + + # Considering users that are indirectly responsible for the dataset (through projects) + possible_datasets = set(user.datasets.all()) + for project in user.projects.all(): + possible_datasets.update(list(project.datasets.all())) + for dataset in possible_datasets: + # Check if the dataset has an access that is about to expire + for access in dataset.accesses.all(): + if ( + access.grant_expires_on + and access.grant_expires_on - day_offset == exec_date + ): + cls.notify(user, access, NotificationVerb.expire) + + @staticmethod + def notify(user: "User", obj: "Access", verb: "NotificationVerb"): + """ + Notifies concerning users about the entity. + """ + offset = user.notification_setting.notification_offset + dispatch_by_email = user.notification_setting.send_email + dispatch_in_app = user.notification_setting.send_in_app + + msg = f"Access for {obj.dataset.title} of the user: {obj.user or obj.contact} is ending in {offset} days." + on = obj.grant_expires_on + + Notification.objects.create( + recipient=user, + verb=verb, + msg=msg, + on=on, + dispatch_by_email=dispatch_by_email, + dispatch_in_app=dispatch_in_app, + content_type=ContentType.objects.get_for_model(obj), + object_id=obj.id, + ).save() + auditlog.register(Access) From ab13bdc6cafcf3927809b045ce739683220606b9 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 2 Nov 2023 18:19:11 +0100 Subject: [PATCH 17/38] add notification logic of the project entity --- core/models/project.py | 70 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/core/models/project.py b/core/models/project.py index a62bc802..d01a567e 100644 --- a/core/models/project.py +++ b/core/models/project.py @@ -1,16 +1,28 @@ +import datetime +import typing +from datetime import timedelta + +from django.conf import settings from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.contenttypes.models import ContentType +from django.contrib.auth import get_user_model from django.db import models +from django.db.models import Q from django.utils.safestring import mark_safe from django.utils.module_loading import import_string from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from core import constants from core.permissions.mapping import PERMISSION_MAPPING +from notification import NotifyMixin +from notification.models import NotificationVerb, Notification, NotificationSetting from .utils import CoreTrackedModel, COMPANY from .partner import HomeOrganisation -from django.conf import settings + +if typing.TYPE_CHECKING: + User = settings.AUTH_USER_MODEL class Project(CoreTrackedModel): @@ -306,6 +318,62 @@ def publish(self): self.elu_accession = generate_id_function(self) self.save() + @staticmethod + def get_notification_recipients(): + """ + Get distinct users that are local custodian of a dataset. + """ + + return get_user_model().objects.filter(Q(projects__isnull=False)).distinct() + + @classmethod + def make_notifications(cls, exec_date: datetime.date): + recipients = cls.get_notification_recipients() + for user in recipients: + notification_setting: NotificationSetting = ( + user.notification_setting or NotificationSetting() + ) + if not ( + notification_setting.send_email or notification_setting.send_in_app + ): + continue + day_offset = timedelta(days=notification_setting.notification_offset) + + for project in user.projects.all(): + # Project start date + if project.start_date and project.start_date - day_offset == exec_date: + cls.notify(user, project, NotificationVerb.start) + # Project end date + if project.end_date and project.end_date - day_offset == exec_date: + cls.notify(user, project, NotificationVerb.end) + + @staticmethod + def notify(user: "User", obj: "Project", verb: "NotificationVerb"): + """ + Notifies concerning users about the entity. + """ + offset = user.notification_setting.notification_offset + dispatch_by_email = user.notification_setting.send_email + dispatch_in_app = user.notification_setting.send_in_app + + if verb == NotificationVerb.start: + msg = f"The project {obj.title} is starting in {offset} days." + on = obj.start_date + else: + msg = f"The project {obj.title} is ending in {offset} days." + on = obj.end_date + + Notification.objects.create( + recipient=user, + verb=verb, + msg=msg, + on=on, + dispatch_by_email=dispatch_by_email, + dispatch_in_app=dispatch_in_app, + content_type=ContentType.objects.get_for_model(obj), + object_id=obj.id, + ).save() + # faster lookup for permissions # https://django-guardian.readthedocs.io/en/stable/userguide/performance.html#direct-foreign-keys From db1907b069778f2689b46e6832a56eaf98b833b5 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 6 Nov 2023 14:21:38 +0100 Subject: [PATCH 18/38] add document notification logic --- core/models/document.py | 73 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/core/models/document.py b/core/models/document.py index 51b7b19b..feba4fb3 100644 --- a/core/models/document.py +++ b/core/models/document.py @@ -1,14 +1,26 @@ import os +import typing +import datetime +from datetime import timedelta +from model_utils import Choices + from django.db import models +from django.conf import settings +from django.db.models import Q +from django.contrib.auth import get_user_model from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.utils import timezone from django.db.models.signals import post_delete from django.dispatch import receiver from django.core.files.storage import default_storage -from model_utils import Choices + from .utils import CoreModel -from core import constants +from notification.models import Notification, NotificationVerb, NotificationSetting +from notification import NotifyMixin + +if typing.TYPE_CHECKING: + User = settings.AUTH_USER_MODEL def get_file_name(instance, filename): @@ -21,7 +33,7 @@ def get_file_name(instance, filename): ) -class Document(CoreModel): +class Document(CoreModel, NotifyMixin): """ Represents a document """ @@ -77,6 +89,61 @@ def shortname(self): def size(self): return self.content.size + @staticmethod + def get_notification_recipients(): + """ + Get distinct users that are local custodian of a dataset. + """ + + return ( + get_user_model() + .objects.filter(Q(projects__isnull=False) | Q(contracts__isnull=False)) + .distinct() + ) + + @classmethod + def make_notifications(cls, exec_date: datetime.date): + recipients = cls.get_notification_recipients() + for user in recipients: + notification_setting: NotificationSetting = ( + user.notification_setting or NotificationSetting() + ) + if not ( + notification_setting.send_email or notification_setting.send_in_app + ): + continue + day_offset = timedelta(days=notification_setting.notification_offset) + + docs = set([p.legal_documents for p in user.projects.all()]) + docs.update([c.legal_documents for c in user.contracts.all()]) + + for doc in docs: + if doc.expiry_date and doc.expiry_date - day_offset == exec_date: + cls.notify(user, doc, NotificationVerb.expire) + + @staticmethod + def notify(user: "User", obj: "Document", verb: "NotificationVerb"): + """ + Notifies concerning users about the entity. + """ + offset = user.notification_setting.notification_offset + dispatch_by_email = user.notification_setting.send_email + dispatch_in_app = user.notification_setting.send_in_app + + msg = f"The Document {obj.shortname} is expiring in {offset} days." + on = obj.expiry_date + + Notification.objects.create( + recipient=user, + verb=verb, + msg=msg, + on=on, + dispatch_by_email=dispatch_by_email, + dispatch_in_app=dispatch_in_app, + content_type=ContentType.objects.get_for_model(obj), + object_id=obj.id, + ).save() + @receiver(post_delete, sender=Document, dispatch_uid="document_delete") def document_cleanup(sender, instance, **kwargs): From 061b192156ba21f7494ef3853c4f93020db9ee04 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 6 Nov 2023 15:47:32 +0100 Subject: [PATCH 19/38] refactor and fix for end-to-end test --- core/models/access.py | 8 ++++---- core/models/dataset.py | 4 ++-- core/models/document.py | 9 +++++---- core/models/project.py | 6 +++--- notification/__init__.py | 14 ++++++++++++++ notification/tasks.py | 11 ++++------- web/admin.py | 7 ++++++- 7 files changed, 38 insertions(+), 21 deletions(-) diff --git a/core/models/access.py b/core/models/access.py index 99bc6e52..4cbb7756 100644 --- a/core/models/access.py +++ b/core/models/access.py @@ -265,8 +265,8 @@ def get_notification_recipients(): def make_notifications(cls, exec_date: date): recipients = cls.get_notification_recipients() for user in recipients: - notification_setting: NotificationSetting = ( - user.notification_setting or NotificationSetting() + notification_setting: NotificationSetting = Access.get_notification_setting( + user ) if not ( notification_setting.send_email or notification_setting.send_in_app @@ -296,13 +296,13 @@ def notify(user: "User", obj: "Access", verb: "NotificationVerb"): dispatch_by_email = user.notification_setting.send_email dispatch_in_app = user.notification_setting.send_in_app - msg = f"Access for {obj.dataset.title} of the user: {obj.user or obj.contact} is ending in {offset} days." + msg = f"Access for {obj.dataset.title} of the user {obj.user or obj.contact} is ending in {offset} days." on = obj.grant_expires_on Notification.objects.create( recipient=user, verb=verb, - msg=msg, + message=msg, on=on, dispatch_by_email=dispatch_by_email, dispatch_in_app=dispatch_in_app, diff --git a/core/models/dataset.py b/core/models/dataset.py index ce018b17..91c5076b 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -255,7 +255,7 @@ def make_notifications(cls, exec_date: datetime.date): recipients = cls.get_notification_recipients() for user in recipients: notification_setting: NotificationSetting = ( - user.notification_setting or NotificationSetting() + Dataset.get_notification_setting(user) ) if not ( notification_setting.send_email or notification_setting.send_in_app @@ -303,7 +303,7 @@ def notify(user: "User", obj: "DataDeclaration", verb: "NotificationVerb"): Notification.objects.create( recipient=user, verb=verb, - msg=msg, + message=msg, on=on, dispatch_by_email=dispatch_by_email, dispatch_in_app=dispatch_in_app, diff --git a/core/models/document.py b/core/models/document.py index feba4fb3..7f285bc5 100644 --- a/core/models/document.py +++ b/core/models/document.py @@ -106,7 +106,7 @@ def make_notifications(cls, exec_date: datetime.date): recipients = cls.get_notification_recipients() for user in recipients: notification_setting: NotificationSetting = ( - user.notification_setting or NotificationSetting() + Document.get_notification_setting(user) ) if not ( notification_setting.send_email or notification_setting.send_in_app @@ -114,8 +114,9 @@ def make_notifications(cls, exec_date: datetime.date): continue day_offset = timedelta(days=notification_setting.notification_offset) - docs = set([p.legal_documents for p in user.projects.all()]) - docs.update([c.legal_documents for c in user.contracts.all()]) + docs = set() + _ = [docs.update(p.legal_documents.all()) for p in user.projects.all()] + _ = [docs.update(c.legal_documents.all()) for c in user.contracts.all()] for doc in docs: if doc.expiry_date and doc.expiry_date - day_offset == exec_date: @@ -136,7 +137,7 @@ def notify(user: "User", obj: "Document", verb: "NotificationVerb"): Notification.objects.create( recipient=user, verb=verb, - msg=msg, + message=msg, on=on, dispatch_by_email=dispatch_by_email, dispatch_in_app=dispatch_in_app, diff --git a/core/models/project.py b/core/models/project.py index d01a567e..5713a80b 100644 --- a/core/models/project.py +++ b/core/models/project.py @@ -25,7 +25,7 @@ User = settings.AUTH_USER_MODEL -class Project(CoreTrackedModel): +class Project(CoreTrackedModel, NotifyMixin): class Meta: app_label = "core" get_latest_by = "added" @@ -331,7 +331,7 @@ def make_notifications(cls, exec_date: datetime.date): recipients = cls.get_notification_recipients() for user in recipients: notification_setting: NotificationSetting = ( - user.notification_setting or NotificationSetting() + Project.get_notification_setting(user) ) if not ( notification_setting.send_email or notification_setting.send_in_app @@ -366,7 +366,7 @@ def notify(user: "User", obj: "Project", verb: "NotificationVerb"): Notification.objects.create( recipient=user, verb=verb, - msg=msg, + message=msg, on=on, dispatch_by_email=dispatch_by_email, dispatch_in_app=dispatch_in_app, diff --git a/notification/__init__.py b/notification/__init__.py index 2ec3f3fa..2dd16d1d 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -48,6 +48,20 @@ def notify(user: "User", obj: object, verb: "NotificationVerb"): f"Subclasses of {NotifyMixin.__name__} must implement {NotifyMixin.notify.__name__}" ) + @staticmethod + def get_notification_setting(user: "User"): + """ + Get the notification setting of the user. + """ + from notification.models import NotificationSetting + + try: + setting = user.notification_setting + except Exception: + setting = NotificationSetting(user=user) + setting.save() + return setting + def get_absolute_url(self) -> str: """ Returns the absolute url of the entity. diff --git a/notification/tasks.py b/notification/tasks.py index c03fd5a1..e2cef8f1 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -1,6 +1,7 @@ from datetime import datetime from collections import defaultdict from datetime import timedelta +import logging from celery import shared_task from django.conf import settings @@ -12,13 +13,7 @@ from notification.models import Notification, NotificationStyle, NotificationVerb from notification import NotifyMixin -# map each notification style to a delta -# delta correspond to the interval + a small delta -NOTIFICATION_MAPPING = { - NotificationStyle.once_per_day: timedelta(days=1, hours=8), - NotificationStyle.once_per_week: timedelta(days=7, hours=16), - NotificationStyle.once_per_month: timedelta(days=33), -} +logger = logging.getLogger(__name__) @shared_task @@ -35,6 +30,8 @@ def create_notifications_for_entities(executation_date: str = None): else: exec_date = datetime.strptime(executation_date, "%Y-%m-%d").date() + logger.info(f"Creating notifications for {exec_date}") + for cls in NotifyMixin.__subclasses__(): cls.make_notifications(exec_date) diff --git a/web/admin.py b/web/admin.py index 4ef8c3d3..6305c23c 100644 --- a/web/admin.py +++ b/web/admin.py @@ -71,9 +71,14 @@ class ProjectAdmin(admin.ModelAdmin): ) +class AccessAdmin(admin.ModelAdmin): + search_fields = ["__str__", "status", "user", "contact", "dataset", "granted_on"] + list_display = ("__str__", "status", "user", "contact", "dataset", "granted_on") + + # DAISY core models admin.site.site_header = "DAISY administration" -admin.site.register(Access) +admin.site.register(Access, AccessAdmin) admin.site.register(Cohort) admin.site.register(Contact, ContactAdmin) admin.site.register(ContactType) From 53da6ea7fd6cf7f7913347cb847ac0758e0aedb8 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 6 Nov 2023 16:17:20 +0100 Subject: [PATCH 20/38] using abc module to enforce the implementation --- notification/__init__.py | 19 +++--- notification/tasks.py | 132 --------------------------------------- 2 files changed, 7 insertions(+), 144 deletions(-) diff --git a/notification/__init__.py b/notification/__init__.py index 2dd16d1d..15c9bc98 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractclassmethod, abstractstaticmethod import typing from typing import List @@ -10,7 +11,7 @@ class NotifyMixin: - @staticmethod + @abstractstaticmethod def get_notification_recipients() -> List["User"]: """ Should Query the users based on their notification settings @@ -19,11 +20,9 @@ def get_notification_recipients() -> List["User"]: Raises: NotImplementedError: It should be implemented by the subclass """ - raise NotImplementedError( - f"Subclasses of {NotifyMixin.__name__} must implement {NotifyMixin.get_notification_recipients.__name__}" - ) + pass - @classmethod + @abstractclassmethod def make_notifications(cls, exec_date: "date"): """ Creates a notifications for the reciepients based on @@ -35,18 +34,14 @@ def make_notifications(cls, exec_date: "date"): Raises: NotImplementedError: It should be implemented by the subclass """ - raise NotImplementedError( - f"Subclasses of {NotifyMixin.__name__} must implement {cls.make_notification.__name__}" - ) + pass - @staticmethod + @abstractstaticmethod def notify(user: "User", obj: object, verb: "NotificationVerb"): """ Notify the user about the entity. """ - raise NotImplementedError( - f"Subclasses of {NotifyMixin.__name__} must implement {NotifyMixin.notify.__name__}" - ) + pass @staticmethod def get_notification_setting(user: "User"): diff --git a/notification/tasks.py b/notification/tasks.py index e2cef8f1..61d80f1d 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -34,135 +34,3 @@ def create_notifications_for_entities(executation_date: str = None): for cls in NotifyMixin.__subclasses__(): cls.make_notifications(exec_date) - - -# @shared_task -# def send_notifications_for_user_by_time(user_id, time): -# """ -# Send a notification report for the current user from the date to the most recent. -# """ -# # get latest notifications for the user, grouped by verb -# notifications = Notification.objects.filter(actor__pk=user_id, time__gte=time) - -# if not notifications: -# return - -# # group notifications per verb -# notifications_by_verb = defaultdict(list) -# for notif in notifications: -# notifications_by_verb[notif.verb].append(notif) - -# user = User.objects.get(pk=user_id) -# context = {"time": time, "user": user, "notifications": dict(notifications_by_verb)} -# send_the_email( -# settings.EMAIL_DONOTREPLY, -# user.email, -# "Notifications", -# "notification/notification_report", -# context, -# ) - - -# @shared_task -# def send_dataset_notification_for_user(user_id, dataset_id, created): -# """ -# Send the notification that a dataset have been updated. -# """ -# dataset = Dataset.objects.get(pk=dataset_id) -# user = User.objects.get(pk=user_id) -# context = {"user": user, "dataset": dataset, "created": created} -# send_the_email( -# settings.EMAIL_DONOTREPLY, -# user.email, -# "Notifications", -# "notification/dataset_notification", -# context, -# ) - - -# @shared_task -# def send_notifications(period): -# """ -# Send notifications for each user based on the period selected. -# Period must be one of `NotificationStyle` but 'every_time'. -# """ -# notification_style = NotificationStyle[period] -# if notification_style is NotificationStyle.every_time: -# raise KeyError("Key not permitted") -# # get users with this setting -# users = User.objects.filter( -# notification_setting__style=notification_style -# ).distinct() -# # map the setting to a timeperiod -# now = timezone.now() -# time = now - NOTIFICATION_MAPPING[notification_style] - -# # get latest notifications -# users = users.filter(notifications__time__gte=time) -# for user in users: -# send_notifications_for_user_by_time.delay(user.id, time) -# return users - - -# @shared_task -# def data_storage_expiry_notifications(): -# now = timezone.now() - -# # the user will receive notifications on two consecutive days prior to storage end date -# window_2_start = now + datetime.timedelta(days=1) -# window_2_end = now + datetime.timedelta(days=2) - -# # the user will receive notifications on two consecutive days, two months prior to storage end date -# window_60_start = now + datetime.timedelta(days=59) -# window_60_end = now + datetime.timedelta(days=60) - -# data_declarations = DataDeclaration.objects.filter( -# Q( -# end_of_storage_duration__gte=window_60_start, -# end_of_storage_duration__lte=window_60_end, -# ) -# | Q( -# end_of_storage_duration__gte=window_2_start, -# end_of_storage_duration__lte=window_2_end, -# ) -# ).order_by("end_of_storage_duration") - -# for ddec in data_declarations: -# for custodian in ddec.dataset.local_custodians.all(): -# Notification.objects.create( -# actor=custodian, -# verb=NotificationVerb.data_storage_expiry, -# content_object=ddec, -# ) - - -# @shared_task -# def document_expiry_notifications(): -# now = timezone.now() - -# # the user will receive notifications on two consecutive days prior to storage end date -# window_2_start = now + datetime.timedelta(days=1) -# window_2_end = now + datetime.timedelta(days=2) - -# # the user will receive notifications on two consecutive days, two months prior to storage end date -# window_60_start = now + datetime.timedelta(days=59) -# window_60_end = now + datetime.timedelta(days=60) - -# documents = Document.objects.filter( -# Q(expiry_date__gte=window_60_start, expiry_date__lte=window_60_end) -# | Q(expiry_date__gte=window_2_start, expiry_date__lte=window_2_end) -# ).order_by("expiry_date") - -# for document in documents: -# print(document.content_type) -# if str(document.content_type) == "project": -# obj = Project.objects.get(pk=document.object_id) -# if str(document.content_type) == "contract": -# obj = Contract.objects.get(pk=document.object_id) -# if obj: -# for custodian in obj.local_custodians.all(): -# Notification.objects.create( -# actor=custodian, -# verb=NotificationVerb.document_expiry, -# content_object=obj, -# ) From 017f2060c00e0865ccc36af9200b230a12661aa8 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Tue, 7 Nov 2023 09:30:53 +0100 Subject: [PATCH 21/38] remove unnecessary imports --- notification/tasks.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/notification/tasks.py b/notification/tasks.py index 61d80f1d..8495fdbe 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -1,16 +1,8 @@ from datetime import datetime -from collections import defaultdict -from datetime import timedelta import logging from celery import shared_task -from django.conf import settings -from django.utils import timezone -from django.db.models import Q -from core.models import Contract, DataDeclaration, Dataset, Document, Project, User -from notification.email_sender import send_the_email -from notification.models import Notification, NotificationStyle, NotificationVerb from notification import NotifyMixin logger = logging.getLogger(__name__) From 0c0fb94a68d198018bc8fa7e5c735cb62a4c3f26 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Tue, 7 Nov 2023 13:41:55 +0100 Subject: [PATCH 22/38] add test for dataset notification logic --- core/tests/notification/test_dataset_notif.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 core/tests/notification/test_dataset_notif.py diff --git a/core/tests/notification/test_dataset_notif.py b/core/tests/notification/test_dataset_notif.py new file mode 100644 index 00000000..a410f22e --- /dev/null +++ b/core/tests/notification/test_dataset_notif.py @@ -0,0 +1,29 @@ +import datetime +from datetime import timedelta + +import pytest + +from test.factories import DatasetFactory, UserFactory, DataDeclarationFactory +from core.models import Dataset +from notification.models import Notification, NotificationSetting + + +@pytest.mark.parametrize("event", ["embargo_date", "end_of_storage_duration"]) +@pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) +@pytest.mark.django_db +def test_dataset_notification_creation(event, offset): + today = datetime.date.today() + event_date = today + timedelta(days=offset) + + user = UserFactory(email="lc@uni.lu") + notif_setting = NotificationSetting(user=user, notification_offset=offset) + notif_setting.save() + + dataset = DatasetFactory(title="Test dataset", local_custodians=[user]) + data_declaration = DataDeclarationFactory(dataset=dataset) + setattr(data_declaration, event, event_date) + data_declaration.save() + + assert Notification.objects.count() == 0 + Dataset.make_notifications(today) + assert Notification.objects.count() == 1 From 21041dcd9226517626ef183ad27e2a8c54452424 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 8 Nov 2023 11:24:02 +0100 Subject: [PATCH 23/38] fix backward relationship between project and local custodians --- ...031_1522.py => 0035_auto_20231108_1041.py} | 12 +++++++++- core/models/access.py | 23 +++++++++++-------- core/models/dataset.py | 4 ++-- core/models/document.py | 4 ++-- core/models/project.py | 6 ++--- 5 files changed, 31 insertions(+), 18 deletions(-) rename core/migrations/{0035_auto_20231031_1522.py => 0035_auto_20231108_1041.py} (60%) diff --git a/core/migrations/0035_auto_20231031_1522.py b/core/migrations/0035_auto_20231108_1041.py similarity index 60% rename from core/migrations/0035_auto_20231031_1522.py rename to core/migrations/0035_auto_20231108_1041.py index 9acb9bb9..dfb493b0 100644 --- a/core/migrations/0035_auto_20231031_1522.py +++ b/core/migrations/0035_auto_20231108_1041.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.20 on 2023-10-31 14:22 +# Generated by Django 3.2.20 on 2023-11-08 09:41 from django.conf import settings from django.db import migrations, models @@ -24,4 +24,14 @@ class Migration(migrations.Migration): verbose_name="User that has the access", ), ), + migrations.AlterField( + model_name="project", + name="local_custodians", + field=models.ManyToManyField( + help_text="Custodians are the local responsibles for the project. This list must include a PI.", + related_name="project_set", + to=settings.AUTH_USER_MODEL, + verbose_name="Local custodians", + ), + ), ] diff --git a/core/models/access.py b/core/models/access.py index 4cbb7756..f8e51a92 100644 --- a/core/models/access.py +++ b/core/models/access.py @@ -252,12 +252,12 @@ def is_active(self): @staticmethod def get_notification_recipients(): """ - Get distinct users that are local custodian of a dataset. + Get distinct users that are local custodian of a dataset or a project. """ return ( get_user_model() - .objects.filter(Q(datasets__isnull=False) | Q(projects__isnull=False)) + .objects.filter(Q(datasets__isnull=False) | Q(project_set__isnull=False)) .distinct() ) @@ -276,16 +276,19 @@ def make_notifications(cls, exec_date: date): # Considering users that are indirectly responsible for the dataset (through projects) possible_datasets = set(user.datasets.all()) - for project in user.projects.all(): + for project in user.project_set.all(): possible_datasets.update(list(project.datasets.all())) - for dataset in possible_datasets: + # Fetch all necessary data at once before the loop + dataset_ids = [dataset.id for dataset in possible_datasets] + accesses = Access.objects.filter(dataset_id__in=dataset_ids) + + for access in accesses: # Check if the dataset has an access that is about to expire - for access in dataset.accesses.all(): - if ( - access.grant_expires_on - and access.grant_expires_on - day_offset == exec_date - ): - cls.notify(user, access, NotificationVerb.expire) + if ( + access.grant_expires_on + and access.grant_expires_on - day_offset == exec_date + ): + cls.notify(user, access, NotificationVerb.expire) @staticmethod def notify(user: "User", obj: "Access", verb: "NotificationVerb"): diff --git a/core/models/dataset.py b/core/models/dataset.py index 91c5076b..23c06d40 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -246,7 +246,7 @@ def get_notification_recipients(): return ( get_user_model() - .objects.filter(Q(datasets__isnull=False) | Q(projects__isnull=False)) + .objects.filter(Q(datasets__isnull=False) | Q(project_set__isnull=False)) .distinct() ) @@ -265,7 +265,7 @@ def make_notifications(cls, exec_date: datetime.date): # Considering users that are indirectly responsible for the dataset (through projects) possible_datasets = set(user.datasets.all()) - for project in user.projects.all(): + for project in user.project_set.all(): possible_datasets.update(list(project.datasets.all())) for dataset in possible_datasets: # Data Declaration (Embargo Date & End of Storage Duration) diff --git a/core/models/document.py b/core/models/document.py index 7f285bc5..34971a53 100644 --- a/core/models/document.py +++ b/core/models/document.py @@ -97,7 +97,7 @@ def get_notification_recipients(): return ( get_user_model() - .objects.filter(Q(projects__isnull=False) | Q(contracts__isnull=False)) + .objects.filter(Q(project_set__isnull=False) | Q(contracts__isnull=False)) .distinct() ) @@ -115,7 +115,7 @@ def make_notifications(cls, exec_date: datetime.date): day_offset = timedelta(days=notification_setting.notification_offset) docs = set() - _ = [docs.update(p.legal_documents.all()) for p in user.projects.all()] + _ = [docs.update(p.legal_documents.all()) for p in user.project_set.all()] _ = [docs.update(c.legal_documents.all()) for c in user.contracts.all()] for doc in docs: diff --git a/core/models/project.py b/core/models/project.py index 5713a80b..6262612b 100644 --- a/core/models/project.py +++ b/core/models/project.py @@ -207,7 +207,7 @@ class AppMeta: local_custodians = models.ManyToManyField( "core.User", - related_name="+", + related_name="project_set", verbose_name="Local custodians", help_text="Custodians are the local responsibles for the project. This list must include a PI.", ) @@ -324,7 +324,7 @@ def get_notification_recipients(): Get distinct users that are local custodian of a dataset. """ - return get_user_model().objects.filter(Q(projects__isnull=False)).distinct() + return get_user_model().objects.filter(Q(project_set__isnull=False)).distinct() @classmethod def make_notifications(cls, exec_date: datetime.date): @@ -339,7 +339,7 @@ def make_notifications(cls, exec_date: datetime.date): continue day_offset = timedelta(days=notification_setting.notification_offset) - for project in user.projects.all(): + for project in user.project_set.all(): # Project start date if project.start_date and project.start_date - day_offset == exec_date: cls.notify(user, project, NotificationVerb.start) From 32135713c81409518fef85f3b50e0ae2eda5111d Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 8 Nov 2023 11:24:58 +0100 Subject: [PATCH 24/38] add tests for access notification logic --- core/tests/notification/test_access_notif.py | 58 +++++++++++++++++++ core/tests/notification/test_dataset_notif.py | 46 ++++++++++++++- core/tests/notification/test_project_notif.py | 0 3 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 core/tests/notification/test_access_notif.py create mode 100644 core/tests/notification/test_project_notif.py diff --git a/core/tests/notification/test_access_notif.py b/core/tests/notification/test_access_notif.py new file mode 100644 index 00000000..220a80e8 --- /dev/null +++ b/core/tests/notification/test_access_notif.py @@ -0,0 +1,58 @@ +import datetime +from datetime import timedelta + +import pytest + +from test.factories import DatasetFactory, UserFactory, ProjectFactory, AccessFactory +from core.models import Access +from notification.models import Notification, NotificationSetting + + +@pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) +@pytest.mark.django_db +def test_access_success_notification_creation(offset): + today = datetime.date.today() + event_date = today + timedelta(days=offset) + + user = UserFactory.create(email="lc@uni.lu") + notif_setting = NotificationSetting(user=user, notification_offset=offset) + notif_setting.save() + + dataset = DatasetFactory.create(title="Test dataset", local_custodians=[user]) + _ = AccessFactory.create(dataset=dataset, grant_expires_on=event_date) + + assert Notification.objects.count() == 0 + Access.make_notifications(today) + assert Notification.objects.count() == 1 + + +@pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) +def test_access_project_lc_notification(offset): + today = datetime.date.today() + event_date = today + timedelta(days=offset) + + dataset_lc = UserFactory.create(email="dataset_lc@uni.lu") + setting1 = NotificationSetting(user=dataset_lc, notification_offset=offset) + setting1.save() + + p1_lc = UserFactory.create(email="p1_lc@uni.lu") + setting2 = NotificationSetting(user=p1_lc, notification_offset=offset) + setting2.save() + + p2_lc = UserFactory.create(email="p2_lc@uni.lu") + setting3 = NotificationSetting(user=p2_lc, notification_offset=offset) + setting3.save() + + project = ProjectFactory.create( + title="Test project", local_custodians=[p1_lc, p2_lc] + ) + dataset = DatasetFactory.create( + title="Test dataset", project=project, local_custodians=[dataset_lc, p1_lc] + ) + _ = AccessFactory.create(dataset=dataset, grant_expires_on=event_date) + + assert Notification.objects.count() == 0 + Access.make_notifications(today) + assert len(Notification.objects.filter(recipient=dataset_lc)) == 1 + assert len(Notification.objects.filter(recipient=p1_lc)) == 1 + assert len(Notification.objects.filter(recipient=p2_lc)) == 1 diff --git a/core/tests/notification/test_dataset_notif.py b/core/tests/notification/test_dataset_notif.py index a410f22e..a7dfbdd3 100644 --- a/core/tests/notification/test_dataset_notif.py +++ b/core/tests/notification/test_dataset_notif.py @@ -3,7 +3,12 @@ import pytest -from test.factories import DatasetFactory, UserFactory, DataDeclarationFactory +from test.factories import ( + DatasetFactory, + UserFactory, + DataDeclarationFactory, + ProjectFactory, +) from core.models import Dataset from notification.models import Notification, NotificationSetting @@ -15,11 +20,11 @@ def test_dataset_notification_creation(event, offset): today = datetime.date.today() event_date = today + timedelta(days=offset) - user = UserFactory(email="lc@uni.lu") + user = UserFactory.create(email="lc@uni.lu") notif_setting = NotificationSetting(user=user, notification_offset=offset) notif_setting.save() - dataset = DatasetFactory(title="Test dataset", local_custodians=[user]) + dataset = DatasetFactory.create(title="Test dataset", local_custodians=[user]) data_declaration = DataDeclarationFactory(dataset=dataset) setattr(data_declaration, event, event_date) data_declaration.save() @@ -27,3 +32,38 @@ def test_dataset_notification_creation(event, offset): assert Notification.objects.count() == 0 Dataset.make_notifications(today) assert Notification.objects.count() == 1 + + +@pytest.mark.parametrize("event", ["embargo_date", "end_of_storage_duration"]) +@pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) +def test_dataset_project_lc_notification(event, offset): + today = datetime.date.today() + event_date = today + timedelta(days=offset) + + dataset_lc = UserFactory.create(email="dataset_lc@uni.lu") + setting1 = NotificationSetting(user=dataset_lc, notification_offset=offset) + setting1.save() + + p1_lc = UserFactory.create(email="p1_lc@uni.lu") + setting2 = NotificationSetting(user=p1_lc, notification_offset=offset) + setting2.save() + + p2_lc = UserFactory.create(email="p2_lc@uni.lu") + setting3 = NotificationSetting(user=p2_lc, notification_offset=offset) + setting3.save() + + project = ProjectFactory.create( + title="Test project", local_custodians=[p1_lc, p2_lc] + ) + dataset = DatasetFactory.create( + title="Test dataset", project=project, local_custodians=[dataset_lc, p1_lc] + ) + data_declaration = DataDeclarationFactory(dataset=dataset) + setattr(data_declaration, event, event_date) + data_declaration.save() + + assert Notification.objects.count() == 0 + Dataset.make_notifications(today) + assert len(Notification.objects.filter(recipient=dataset_lc)) == 1 + assert len(Notification.objects.filter(recipient=p1_lc)) == 1 + assert len(Notification.objects.filter(recipient=p2_lc)) == 1 diff --git a/core/tests/notification/test_project_notif.py b/core/tests/notification/test_project_notif.py new file mode 100644 index 00000000..e69de29b From cb3e1d6fce6102c8799b92780e6fd8af6822d1a6 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 8 Nov 2023 11:59:28 +0100 Subject: [PATCH 25/38] add more tests for edge cases and anti paths --- core/models/access.py | 13 +++-- core/tests/notification/test_access_notif.py | 54 ++++++++++++++++++- core/tests/notification/test_dataset_notif.py | 25 +++++++++ 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/core/models/access.py b/core/models/access.py index f8e51a92..5cf72d05 100644 --- a/core/models/access.py +++ b/core/models/access.py @@ -276,11 +276,18 @@ def make_notifications(cls, exec_date: date): # Considering users that are indirectly responsible for the dataset (through projects) possible_datasets = set(user.datasets.all()) - for project in user.project_set.all(): - possible_datasets.update(list(project.datasets.all())) + possible_datasets.update( + [ + dataset + for project in user.project_set.all() + for dataset in project.datasets.all() + ] + ) # Fetch all necessary data at once before the loop dataset_ids = [dataset.id for dataset in possible_datasets] - accesses = Access.objects.filter(dataset_id__in=dataset_ids) + accesses = Access.objects.filter( + Q(dataset_id__in=dataset_ids) & Q(status=StatusChoices.active) + ) for access in accesses: # Check if the dataset has an access that is about to expire diff --git a/core/tests/notification/test_access_notif.py b/core/tests/notification/test_access_notif.py index 220a80e8..5c391f27 100644 --- a/core/tests/notification/test_access_notif.py +++ b/core/tests/notification/test_access_notif.py @@ -5,6 +5,7 @@ from test.factories import DatasetFactory, UserFactory, ProjectFactory, AccessFactory from core.models import Access +from core.models.access import StatusChoices from notification.models import Notification, NotificationSetting @@ -19,13 +20,54 @@ def test_access_success_notification_creation(offset): notif_setting.save() dataset = DatasetFactory.create(title="Test dataset", local_custodians=[user]) - _ = AccessFactory.create(dataset=dataset, grant_expires_on=event_date) + _ = AccessFactory.create( + dataset=dataset, grant_expires_on=event_date, status=StatusChoices.active + ) assert Notification.objects.count() == 0 Access.make_notifications(today) assert Notification.objects.count() == 1 +def test_accesss_unmatching_dates(): + today = datetime.date.today() + event_date = today + timedelta(days=20) + + user = UserFactory.create(email="lc@uni.lu") + notif_setting = NotificationSetting(user=user, notification_offset=30) + notif_setting.save() + + dataset = DatasetFactory.create(title="Test dataset", local_custodians=[user]) + _ = AccessFactory.create( + dataset=dataset, grant_expires_on=event_date, status=StatusChoices.active + ) + + assert Notification.objects.count() == 0 + Access.make_notifications(today) + assert Notification.objects.count() == 0 + + +@pytest.mark.parametrize( + "state", + [StatusChoices.precreated, StatusChoices.suspended, StatusChoices.terminated], +) +@pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) +def test_access_no_notif_inactive_access(state, offset): + today = datetime.date.today() + event_date = today + timedelta(days=offset) + + user = UserFactory.create(email="lc@uni.lu") + notif_setting = NotificationSetting(user=user, notification_offset=offset) + notif_setting.save() + + dataset = DatasetFactory.create(title="Test dataset", local_custodians=[user]) + _ = AccessFactory.create(dataset=dataset, grant_expires_on=event_date, status=state) + + assert Notification.objects.count() == 0 + Access.make_notifications(today) + assert Notification.objects.count() == 0 + + @pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) def test_access_project_lc_notification(offset): today = datetime.date.today() @@ -49,10 +91,18 @@ def test_access_project_lc_notification(offset): dataset = DatasetFactory.create( title="Test dataset", project=project, local_custodians=[dataset_lc, p1_lc] ) - _ = AccessFactory.create(dataset=dataset, grant_expires_on=event_date) + _ = AccessFactory.create( + dataset=dataset, grant_expires_on=event_date, status=StatusChoices.active + ) assert Notification.objects.count() == 0 Access.make_notifications(today) assert len(Notification.objects.filter(recipient=dataset_lc)) == 1 assert len(Notification.objects.filter(recipient=p1_lc)) == 1 assert len(Notification.objects.filter(recipient=p2_lc)) == 1 + + +def test_access_handles_no_recipients(): + exec_date = datetime.date.today() + Access.make_notifications(exec_date) + assert Notification.objects.count() == 0 diff --git a/core/tests/notification/test_dataset_notif.py b/core/tests/notification/test_dataset_notif.py index a7dfbdd3..0c197e40 100644 --- a/core/tests/notification/test_dataset_notif.py +++ b/core/tests/notification/test_dataset_notif.py @@ -34,6 +34,25 @@ def test_dataset_notification_creation(event, offset): assert Notification.objects.count() == 1 +@pytest.mark.parametrize("event", ["embargo_date", "end_of_storage_duration"]) +def test_dataset_unmatching_dates(event): + today = datetime.date.today() + event_date = today + timedelta(days=20) + + user = UserFactory.create(email="lc@uni.lu") + notif_setting = NotificationSetting(user=user, notification_offset=30) + notif_setting.save() + + dataset = DatasetFactory.create(title="Test dataset", local_custodians=[user]) + data_declaration = DataDeclarationFactory(dataset=dataset) + setattr(data_declaration, event, event_date) + data_declaration.save() + + assert Notification.objects.count() == 0 + Dataset.make_notifications(today) + assert Notification.objects.count() == 0 + + @pytest.mark.parametrize("event", ["embargo_date", "end_of_storage_duration"]) @pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) def test_dataset_project_lc_notification(event, offset): @@ -67,3 +86,9 @@ def test_dataset_project_lc_notification(event, offset): assert len(Notification.objects.filter(recipient=dataset_lc)) == 1 assert len(Notification.objects.filter(recipient=p1_lc)) == 1 assert len(Notification.objects.filter(recipient=p2_lc)) == 1 + + +def test_dataset_handles_no_recipients(): + exec_date = datetime.date.today() + Dataset.make_notifications(exec_date) + assert Notification.objects.count() == 0 From 3797fe1fbf21d9ebb28a48976fdaa1facfeb8cc7 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 8 Nov 2023 14:01:38 +0100 Subject: [PATCH 26/38] add tests for document notifications --- core/models/document.py | 5 +- core/models/project.py | 2 +- .../tests/notification/test_document_notif.py | 93 +++++++++++++++++++ core/tests/notification/test_project_notif.py | 55 +++++++++++ 4 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 core/tests/notification/test_document_notif.py diff --git a/core/models/document.py b/core/models/document.py index 34971a53..60d855fd 100644 --- a/core/models/document.py +++ b/core/models/document.py @@ -92,7 +92,7 @@ def size(self): @staticmethod def get_notification_recipients(): """ - Get distinct users that are local custodian of a dataset. + Get distinct users that are local custodian of a project or a contract. """ return ( @@ -117,6 +117,9 @@ def make_notifications(cls, exec_date: datetime.date): docs = set() _ = [docs.update(p.legal_documents.all()) for p in user.project_set.all()] _ = [docs.update(c.legal_documents.all()) for c in user.contracts.all()] + # Also add all the documents of all contracts of all projects to address the indirect LCs of parent projects + for p in user.project_set.all(): + _ = [docs.update(c.legal_documents.all()) for c in p.contracts.all()] for doc in docs: if doc.expiry_date and doc.expiry_date - day_offset == exec_date: diff --git a/core/models/project.py b/core/models/project.py index 6262612b..fc652419 100644 --- a/core/models/project.py +++ b/core/models/project.py @@ -321,7 +321,7 @@ def publish(self): @staticmethod def get_notification_recipients(): """ - Get distinct users that are local custodian of a dataset. + Get distinct users that are local custodian of a project. """ return get_user_model().objects.filter(Q(project_set__isnull=False)).distinct() diff --git a/core/tests/notification/test_document_notif.py b/core/tests/notification/test_document_notif.py new file mode 100644 index 00000000..9799e6e5 --- /dev/null +++ b/core/tests/notification/test_document_notif.py @@ -0,0 +1,93 @@ +import datetime +from datetime import timedelta + +import pytest + +from test.factories import ( + ProjectDocumentFactory, + ContractDocumentFactory, + ContractFactory, + UserFactory, + ProjectFactory, +) +from core.models import Document +from notification.models import Notification, NotificationSetting + + +@pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) +@pytest.mark.django_db +def test_document_notification_creation(offset): + today = datetime.date.today() + event_date = today + timedelta(days=offset) + + user = UserFactory.create(email="lc@uni.lu") + notif_setting = NotificationSetting(user=user, notification_offset=offset) + notif_setting.save() + + project = ProjectFactory.create(title="Test project", local_custodians=[user]) + _ = ProjectDocumentFactory.create(content_object=project, expiry_date=event_date) + + contract = ContractFactory.create(local_custodians=[user]) + _ = ContractDocumentFactory.create(content_object=contract, expiry_date=event_date) + + assert Notification.objects.count() == 0 + Document.make_notifications(today) + assert Notification.objects.count() == 2 + + +def test_document_unmatching_dates(): + today = datetime.date.today() + event_date = today + timedelta(days=10) + + user = UserFactory.create(email="lc@uni.lu") + notif_setting = NotificationSetting(user=user, notification_offset=30) + notif_setting.save() + + project = ProjectFactory.create(title="Test project", local_custodians=[user]) + _ = ProjectDocumentFactory.create(content_object=project, expiry_date=event_date) + + contract = ContractFactory.create(local_custodians=[user]) + _ = ContractDocumentFactory.create(content_object=contract, expiry_date=event_date) + + assert Notification.objects.count() == 0 + Document.make_notifications(today) + assert Notification.objects.count() == 0 + + +@pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) +def test_document_project_lc_notification(offset): + today = datetime.date.today() + event_date = today + timedelta(days=offset) + + contract_lc = UserFactory.create(email="contract_lc@uni.lu") + setting1 = NotificationSetting(user=contract_lc, notification_offset=offset) + setting1.save() + + p1_lc = UserFactory.create(email="p1_lc@uni.lu") + setting2 = NotificationSetting(user=p1_lc, notification_offset=offset) + setting2.save() + + p2_lc = UserFactory.create(email="p2_lc@uni.lu") + setting3 = NotificationSetting(user=p2_lc, notification_offset=offset) + setting3.save() + + project = ProjectFactory.create( + title="Test project", local_custodians=[p1_lc, p2_lc] + ) + contract = ContractFactory.create( + project=project, local_custodians=[contract_lc, p1_lc] + ) + + _ = ContractDocumentFactory.create(content_object=contract, expiry_date=event_date) + + assert Notification.objects.count() == 0 + Document.make_notifications(today) + assert len(Notification.objects.filter(recipient=contract_lc)) == 1 + assert len(Notification.objects.filter(recipient=p1_lc)) == 1 + assert len(Notification.objects.filter(recipient=p2_lc)) == 1 + + +def test_document_handles_no_recipients(): + exec_date = datetime.date.today() + Document.make_notifications(exec_date) + assert Notification.objects.count() == 0 diff --git a/core/tests/notification/test_project_notif.py b/core/tests/notification/test_project_notif.py index e69de29b..16f4501d 100644 --- a/core/tests/notification/test_project_notif.py +++ b/core/tests/notification/test_project_notif.py @@ -0,0 +1,55 @@ +import datetime +from datetime import timedelta + +import pytest + +from test.factories import ( + UserFactory, + ProjectFactory, +) +from core.models import Project +from notification.models import Notification, NotificationSetting + + +@pytest.mark.parametrize("event", ["start_date", "end_date"]) +@pytest.mark.parametrize("offset", [1, 10, 30, 60, 90]) +@pytest.mark.django_db +def test_project_notification_creation(event, offset): + today = datetime.date.today() + event_date = today + timedelta(days=offset) + + user = UserFactory.create(email="lc@uni.lu") + notif_setting = NotificationSetting(user=user, notification_offset=offset) + notif_setting.save() + + project = ProjectFactory.create(title="Test project", local_custodians=[user]) + setattr(project, event, event_date) + project.save() + + assert Notification.objects.count() == 0 + Project.make_notifications(today) + assert Notification.objects.count() == 1 + + +@pytest.mark.parametrize("event", ["start_date", "end_date"]) +def test_project_unmatching_dates(event): + today = datetime.date.today() + event_date = today + timedelta(days=20) + + user = UserFactory.create(email="lc@uni.lu") + notif_setting = NotificationSetting(user=user, notification_offset=30) + notif_setting.save() + + project = ProjectFactory.create(title="Test project", local_custodians=[user]) + setattr(project, event, event_date) + project.save() + + assert Notification.objects.count() == 0 + Project.make_notifications(today) + assert Notification.objects.count() == 0 + + +def test_project_handles_no_recipients(): + exec_date = datetime.date.today() + Project.make_notifications(exec_date) + assert Notification.objects.count() == 0 From 12dd5d5675eaef751f165428a70bec7935e19960 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Wed, 8 Nov 2023 14:13:15 +0100 Subject: [PATCH 27/38] modify docstring --- core/models/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/models/dataset.py b/core/models/dataset.py index 23c06d40..8ff26fc3 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -241,7 +241,7 @@ def publish(self, save=True): @staticmethod def get_notification_recipients(): """ - Get distinct users that are local custodian of a dataset. + Get distinct users that are local custodian of a dataset or a project. """ return ( From d6820db0bbde042938a29db1a38de56c74cd3fca Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 9 Nov 2023 14:24:52 +0100 Subject: [PATCH 28/38] fix typo in task name --- notification/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/notification/tasks.py b/notification/tasks.py index 8495fdbe..8a0f2ea4 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -9,7 +9,7 @@ @shared_task -def create_notifications_for_entities(executation_date: str = None): +def create_notifications_for_entities(execution_date: str = None): """ Loops Through all the entitie that implement the Notificaiton Mixin and creates a notification for each one of them according to the logic. @@ -17,10 +17,10 @@ def create_notifications_for_entities(executation_date: str = None): Params: executation_date: The date of the execution of the task. FORMAT: YYYY-MM-DD (DEFAULT: Today) """ - if not executation_date: + if not execution_date: exec_date = datetime.now().date() else: - exec_date = datetime.strptime(executation_date, "%Y-%m-%d").date() + exec_date = datetime.strptime(execution_date, "%Y-%m-%d").date() logger.info(f"Creating notifications for {exec_date}") From 2968ecf098b98b35a6a848b560d8d0be43bfbb85 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 9 Nov 2023 14:29:16 +0100 Subject: [PATCH 29/38] separate abstract and static decorators --- notification/__init__.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/notification/__init__.py b/notification/__init__.py index 15c9bc98..13c851eb 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,6 +1,6 @@ -from abc import ABC, abstractclassmethod, abstractstaticmethod +from abc import ABC, abstractmethod import typing -from typing import List +from typing import List, Optional if typing.TYPE_CHECKING: from django.conf import settings @@ -10,8 +10,9 @@ User = settings.AUTH_USER_MODEL -class NotifyMixin: - @abstractstaticmethod +class NotifyMixin(ABC): + @staticmethod + @abstractmethod def get_notification_recipients() -> List["User"]: """ Should Query the users based on their notification settings @@ -22,7 +23,8 @@ def get_notification_recipients() -> List["User"]: """ pass - @abstractclassmethod + @classmethod + @abstractmethod def make_notifications(cls, exec_date: "date"): """ Creates a notifications for the reciepients based on @@ -36,7 +38,8 @@ def make_notifications(cls, exec_date: "date"): """ pass - @abstractstaticmethod + @staticmethod + @abstractmethod def notify(user: "User", obj: object, verb: "NotificationVerb"): """ Notify the user about the entity. @@ -57,7 +60,7 @@ def get_notification_setting(user: "User"): setting.save() return setting - def get_absolute_url(self) -> str: + def get_absolute_url(self) -> Optional[str]: """ Returns the absolute url of the entity. """ From 15b050c786c5a633b136b2f82417fc8a5f37b377 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 9 Nov 2023 14:34:56 +0100 Subject: [PATCH 30/38] fixing typos --- notification/__init__.py | 12 +++--------- notification/tasks.py | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/notification/__init__.py b/notification/__init__.py index 13c851eb..a4d3cd20 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -15,11 +15,8 @@ class NotifyMixin(ABC): @abstractmethod def get_notification_recipients() -> List["User"]: """ - Should Query the users based on their notification settings + Should query the users based on their notification settings and the entity. - - Raises: - NotImplementedError: It should be implemented by the subclass """ pass @@ -27,14 +24,11 @@ def get_notification_recipients() -> List["User"]: @abstractmethod def make_notifications(cls, exec_date: "date"): """ - Creates a notifications for the reciepients based on + Creates notifications for the reciepients based on the business logic of the entity. Params: - exec_date: The date of execution - - Raises: - NotImplementedError: It should be implemented by the subclass + exec_date: The date of execution of the task. """ pass diff --git a/notification/tasks.py b/notification/tasks.py index 8a0f2ea4..e452a815 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -11,8 +11,8 @@ @shared_task def create_notifications_for_entities(execution_date: str = None): """ - Loops Through all the entitie that implement the Notificaiton Mixin - and creates a notification for each one of them according to the logic. + Loops through all the entities that implement the Notificaiton Mixin + and creates notifications for each one of them according to the logic. Params: executation_date: The date of the execution of the task. FORMAT: YYYY-MM-DD (DEFAULT: Today) From ec0701f8fb5792abae14e68fe8ff34c8b782a220 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 9 Nov 2023 14:42:44 +0100 Subject: [PATCH 31/38] revert change to the celery app config file --- elixir_daisy/celery_app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elixir_daisy/celery_app.py b/elixir_daisy/celery_app.py index 790e64f5..1632732d 100644 --- a/elixir_daisy/celery_app.py +++ b/elixir_daisy/celery_app.py @@ -2,6 +2,8 @@ from celery import Celery +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "elixir_daisy.settings") + app = Celery("daisy") # Using a string here means the worker doesn't have to serialize From 14fbdcab73a1d664e56c8dcb9a3d488b20e9031d Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 9 Nov 2023 15:31:31 +0100 Subject: [PATCH 32/38] remove ABC --- notification/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notification/__init__.py b/notification/__init__.py index a4d3cd20..ebdeae90 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod +from abc import abstractmethod import typing from typing import List, Optional @@ -10,7 +10,7 @@ User = settings.AUTH_USER_MODEL -class NotifyMixin(ABC): +class NotifyMixin: @staticmethod @abstractmethod def get_notification_recipients() -> List["User"]: From 84e90badb5c132e41649546c4ecb9a3ee5f38e77 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Thu, 9 Nov 2023 15:45:41 +0100 Subject: [PATCH 33/38] solve the metaclass problem --- core/models/access.py | 4 ++-- core/models/dataset.py | 4 ++-- core/models/document.py | 4 ++-- core/models/project.py | 4 ++-- core/models/utils.py | 6 ++++++ notification/__init__.py | 4 ++-- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/core/models/access.py b/core/models/access.py index 5cf72d05..83e67a75 100644 --- a/core/models/access.py +++ b/core/models/access.py @@ -13,7 +13,7 @@ from enumchoicefield import EnumChoiceField, ChoiceEnum -from .utils import CoreModel +from .utils import CoreModel, CoreNotifyMeta from notification import NotifyMixin from notification.models import NotificationSetting, NotificationVerb, Notification @@ -34,7 +34,7 @@ class StatusChoices(ChoiceEnum): terminated = "Terminated" -class Access(CoreModel, NotifyMixin): +class Access(CoreModel, NotifyMixin, metaclass=CoreNotifyMeta): """ Represents the access given to an internal (LCSB) entity over data storage locations. """ diff --git a/core/models/dataset.py b/core/models/dataset.py index 8ff26fc3..5e5bcc5f 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -17,14 +17,14 @@ from core.permissions.mapping import PERMISSION_MAPPING from notification import NotifyMixin from notification.models import Notification, NotificationVerb, NotificationSetting -from .utils import CoreTrackedModel, TextFieldWithInputWidget +from .utils import CoreTrackedModel, TextFieldWithInputWidget, CoreNotifyMeta from .partner import HomeOrganisation if typing.TYPE_CHECKING: User = settings.AUTH_USER_MODEL -class Dataset(CoreTrackedModel, NotifyMixin): +class Dataset(CoreTrackedModel, NotifyMixin, metaclass=CoreNotifyMeta): class Meta: app_label = "core" get_latest_by = "added" diff --git a/core/models/document.py b/core/models/document.py index 60d855fd..8b2a8646 100644 --- a/core/models/document.py +++ b/core/models/document.py @@ -15,7 +15,7 @@ from django.dispatch import receiver from django.core.files.storage import default_storage -from .utils import CoreModel +from .utils import CoreModel, CoreNotifyMeta from notification.models import Notification, NotificationVerb, NotificationSetting from notification import NotifyMixin @@ -33,7 +33,7 @@ def get_file_name(instance, filename): ) -class Document(CoreModel, NotifyMixin): +class Document(CoreModel, NotifyMixin, metaclass=CoreNotifyMeta): """ Represents a document """ diff --git a/core/models/project.py b/core/models/project.py index fc652419..ec47a1fe 100644 --- a/core/models/project.py +++ b/core/models/project.py @@ -17,7 +17,7 @@ from notification import NotifyMixin from notification.models import NotificationVerb, Notification, NotificationSetting -from .utils import CoreTrackedModel, COMPANY +from .utils import CoreTrackedModel, COMPANY, CoreNotifyMeta from .partner import HomeOrganisation @@ -25,7 +25,7 @@ User = settings.AUTH_USER_MODEL -class Project(CoreTrackedModel, NotifyMixin): +class Project(CoreTrackedModel, NotifyMixin, metaclass=CoreNotifyMeta): class Meta: app_label = "core" get_latest_by = "added" diff --git a/core/models/utils.py b/core/models/utils.py index 6f2bd02c..fbdb4515 100644 --- a/core/models/utils.py +++ b/core/models/utils.py @@ -10,6 +10,8 @@ from django.utils.module_loading import import_string from django.contrib.auth.hashers import make_password +from notification import NotifyMixin + COMPANY = getattr(settings, "COMPANY", "Company") @@ -46,6 +48,10 @@ class Meta: abstract = True +class CoreNotifyMeta(type(CoreModel), type(NotifyMixin)): + pass + + class CoreTrackedModel(CoreModel): elu_accession = models.CharField( unique=True, diff --git a/notification/__init__.py b/notification/__init__.py index ebdeae90..a4d3cd20 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,4 +1,4 @@ -from abc import abstractmethod +from abc import ABC, abstractmethod import typing from typing import List, Optional @@ -10,7 +10,7 @@ User = settings.AUTH_USER_MODEL -class NotifyMixin: +class NotifyMixin(ABC): @staticmethod @abstractmethod def get_notification_recipients() -> List["User"]: From 8a6621aee4c01689841c4e8ecfbcf684cd6e888b Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Fri, 10 Nov 2023 17:10:00 +0100 Subject: [PATCH 34/38] refactor and abstract the user loop --- core/models/access.py | 57 +++++++++++++++++----------------------- core/models/dataset.py | 51 +++++++++++++++-------------------- core/models/document.py | 35 +++++++++--------------- core/models/project.py | 29 +++++++------------- notification/__init__.py | 25 +++++++++++++++++- 5 files changed, 92 insertions(+), 105 deletions(-) diff --git a/core/models/access.py b/core/models/access.py index 83e67a75..1dd31df5 100644 --- a/core/models/access.py +++ b/core/models/access.py @@ -262,40 +262,31 @@ def get_notification_recipients(): ) @classmethod - def make_notifications(cls, exec_date: date): - recipients = cls.get_notification_recipients() - for user in recipients: - notification_setting: NotificationSetting = Access.get_notification_setting( - user - ) - if not ( - notification_setting.send_email or notification_setting.send_in_app - ): - continue - day_offset = timedelta(days=notification_setting.notification_offset) - - # Considering users that are indirectly responsible for the dataset (through projects) - possible_datasets = set(user.datasets.all()) - possible_datasets.update( - [ - dataset - for project in user.project_set.all() - for dataset in project.datasets.all() - ] - ) - # Fetch all necessary data at once before the loop - dataset_ids = [dataset.id for dataset in possible_datasets] - accesses = Access.objects.filter( - Q(dataset_id__in=dataset_ids) & Q(status=StatusChoices.active) - ) + def make_notifications_for_user( + cls, day_offset: timedelta, exec_date: date, user: "User" + ): + # Considering users that are indirectly responsible for the dataset (through projects) + possible_datasets = set(user.datasets.all()) + possible_datasets.update( + [ + dataset + for project in user.project_set.all() + for dataset in project.datasets.all() + ] + ) + # Fetch all necessary data at once before the loop + dataset_ids = [dataset.id for dataset in possible_datasets] + accesses = Access.objects.filter( + Q(dataset_id__in=dataset_ids) & Q(status=StatusChoices.active) + ) - for access in accesses: - # Check if the dataset has an access that is about to expire - if ( - access.grant_expires_on - and access.grant_expires_on - day_offset == exec_date - ): - cls.notify(user, access, NotificationVerb.expire) + for access in accesses: + # Check if the dataset has an access that is about to expire + if ( + access.grant_expires_on + and access.grant_expires_on - day_offset == exec_date + ): + cls.notify(user, access, NotificationVerb.expire) @staticmethod def notify(user: "User", obj: "Access", verb: "NotificationVerb"): diff --git a/core/models/dataset.py b/core/models/dataset.py index 5e5bcc5f..1dafdcb9 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -251,36 +251,27 @@ def get_notification_recipients(): ) @classmethod - def make_notifications(cls, exec_date: datetime.date): - recipients = cls.get_notification_recipients() - for user in recipients: - notification_setting: NotificationSetting = ( - Dataset.get_notification_setting(user) - ) - if not ( - notification_setting.send_email or notification_setting.send_in_app - ): - continue - day_offset = timedelta(days=notification_setting.notification_offset) - - # Considering users that are indirectly responsible for the dataset (through projects) - possible_datasets = set(user.datasets.all()) - for project in user.project_set.all(): - possible_datasets.update(list(project.datasets.all())) - for dataset in possible_datasets: - # Data Declaration (Embargo Date & End of Storage Duration) - for data_declaration in dataset.data_declarations.all(): - if ( - data_declaration.embargo_date - and data_declaration.embargo_date - day_offset == exec_date - ): - cls.notify(user, data_declaration, NotificationVerb.embargo_end) - if ( - data_declaration.end_of_storage_duration - and data_declaration.end_of_storage_duration - day_offset - == exec_date - ): - cls.notify(user, data_declaration, NotificationVerb.end) + def make_notifications_for_user( + cls, day_offset: timedelta, exec_date: datetime.date, user: "User" + ): + # Considering users that are indirectly responsible for the dataset (through projects) + possible_datasets = set(user.datasets.all()) + for project in user.project_set.all(): + possible_datasets.update(list(project.datasets.all())) + for dataset in possible_datasets: + # Data Declaration (Embargo Date & End of Storage Duration) + for data_declaration in dataset.data_declarations.all(): + if ( + data_declaration.embargo_date + and data_declaration.embargo_date - day_offset == exec_date + ): + cls.notify(user, data_declaration, NotificationVerb.embargo_end) + if ( + data_declaration.end_of_storage_duration + and data_declaration.end_of_storage_duration - day_offset + == exec_date + ): + cls.notify(user, data_declaration, NotificationVerb.end) @staticmethod def notify(user: "User", obj: "DataDeclaration", verb: "NotificationVerb"): diff --git a/core/models/document.py b/core/models/document.py index 8b2a8646..09469752 100644 --- a/core/models/document.py +++ b/core/models/document.py @@ -102,28 +102,19 @@ def get_notification_recipients(): ) @classmethod - def make_notifications(cls, exec_date: datetime.date): - recipients = cls.get_notification_recipients() - for user in recipients: - notification_setting: NotificationSetting = ( - Document.get_notification_setting(user) - ) - if not ( - notification_setting.send_email or notification_setting.send_in_app - ): - continue - day_offset = timedelta(days=notification_setting.notification_offset) - - docs = set() - _ = [docs.update(p.legal_documents.all()) for p in user.project_set.all()] - _ = [docs.update(c.legal_documents.all()) for c in user.contracts.all()] - # Also add all the documents of all contracts of all projects to address the indirect LCs of parent projects - for p in user.project_set.all(): - _ = [docs.update(c.legal_documents.all()) for c in p.contracts.all()] - - for doc in docs: - if doc.expiry_date and doc.expiry_date - day_offset == exec_date: - cls.notify(user, doc, NotificationVerb.expire) + def make_notifications_for_user( + cls, day_offset: timedelta, exec_date: datetime.date, user: "User" + ): + docs = set() + [docs.update(p.legal_documents.all()) for p in user.project_set.all()] + [docs.update(c.legal_documents.all()) for c in user.contracts.all()] + # Also add all the documents of all contracts of all projects to address the indirect LCs of parent projects + for p in user.project_set.all(): + _ = [docs.update(c.legal_documents.all()) for c in p.contracts.all()] + + for doc in docs: + if doc.expiry_date and doc.expiry_date - day_offset == exec_date: + cls.notify(user, doc, NotificationVerb.expire) @staticmethod def notify(user: "User", obj: "Document", verb: "NotificationVerb"): diff --git a/core/models/project.py b/core/models/project.py index ec47a1fe..a352914a 100644 --- a/core/models/project.py +++ b/core/models/project.py @@ -327,25 +327,16 @@ def get_notification_recipients(): return get_user_model().objects.filter(Q(project_set__isnull=False)).distinct() @classmethod - def make_notifications(cls, exec_date: datetime.date): - recipients = cls.get_notification_recipients() - for user in recipients: - notification_setting: NotificationSetting = ( - Project.get_notification_setting(user) - ) - if not ( - notification_setting.send_email or notification_setting.send_in_app - ): - continue - day_offset = timedelta(days=notification_setting.notification_offset) - - for project in user.project_set.all(): - # Project start date - if project.start_date and project.start_date - day_offset == exec_date: - cls.notify(user, project, NotificationVerb.start) - # Project end date - if project.end_date and project.end_date - day_offset == exec_date: - cls.notify(user, project, NotificationVerb.end) + def make_notifications_for_user( + cls, day_offset: timedelta, exec_date: datetime.date, user: "User" + ): + for project in user.project_set.all(): + # Project start date + if project.start_date and project.start_date - day_offset == exec_date: + cls.notify(user, project, NotificationVerb.start) + # Project end date + if project.end_date and project.end_date - day_offset == exec_date: + cls.notify(user, project, NotificationVerb.end) @staticmethod def notify(user: "User", obj: "Project", verb: "NotificationVerb"): diff --git a/notification/__init__.py b/notification/__init__.py index a4d3cd20..2eddb5a9 100644 --- a/notification/__init__.py +++ b/notification/__init__.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod import typing from typing import List, Optional +from datetime import timedelta if typing.TYPE_CHECKING: from django.conf import settings @@ -21,7 +22,6 @@ def get_notification_recipients() -> List["User"]: pass @classmethod - @abstractmethod def make_notifications(cls, exec_date: "date"): """ Creates notifications for the reciepients based on @@ -30,6 +30,29 @@ def make_notifications(cls, exec_date: "date"): Params: exec_date: The date of execution of the task. """ + recipients = cls.get_notification_recipients() + for user in recipients: + notification_setting = cls.get_notification_setting(user) + if not ( + notification_setting.send_email or notification_setting.send_in_app + ): + continue + day_offset = timedelta(days=notification_setting.notification_offset) + cls.make_notifications_for_user(day_offset, exec_date, user) + + @classmethod + @abstractmethod + def make_notifications_for_user( + cls, day_offset: "timedelta", exec_date: "date", user: "User" + ): + """ + Creates notifications for the user based on the business logic of the entity. + + Params: + day_offset: The offset of the notification. + exec_date: The date of execution of the task. + user: The user to create the notification for. + """ pass @staticmethod From 307744465e6b7e5264b16dc0711c54619a08adf7 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Fri, 10 Nov 2023 17:54:29 +0100 Subject: [PATCH 35/38] add more logging around notifications --- core/models/access.py | 7 +++++-- core/models/dataset.py | 8 +++++++- core/models/document.py | 8 +++++++- core/models/project.py | 8 +++++++- notification/tasks.py | 1 + 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/core/models/access.py b/core/models/access.py index 1dd31df5..1ba5b068 100644 --- a/core/models/access.py +++ b/core/models/access.py @@ -15,7 +15,8 @@ from .utils import CoreModel, CoreNotifyMeta from notification import NotifyMixin -from notification.models import NotificationSetting, NotificationVerb, Notification +from notification.models import NotificationVerb, Notification +from core.utils import DaisyLogger from auditlog.registry import auditlog from auditlog.models import AuditlogHistoryField @@ -24,7 +25,7 @@ User = settings.AUTH_USER_MODEL -logger = logging.getLogger(__name__) +logger = DaisyLogger(__name__) class StatusChoices(ChoiceEnum): @@ -300,6 +301,8 @@ def notify(user: "User", obj: "Access", verb: "NotificationVerb"): msg = f"Access for {obj.dataset.title} of the user {obj.user or obj.contact} is ending in {offset} days." on = obj.grant_expires_on + logger.info(f"Creating a notification for {user} : {msg}") + Notification.objects.create( recipient=user, verb=verb, diff --git a/core/models/dataset.py b/core/models/dataset.py index 1dafdcb9..b7ed550a 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -13,10 +13,11 @@ from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from core import constants +from core.utils import DaisyLogger from core.models import DataDeclaration from core.permissions.mapping import PERMISSION_MAPPING from notification import NotifyMixin -from notification.models import Notification, NotificationVerb, NotificationSetting +from notification.models import Notification, NotificationVerb from .utils import CoreTrackedModel, TextFieldWithInputWidget, CoreNotifyMeta from .partner import HomeOrganisation @@ -24,6 +25,9 @@ User = settings.AUTH_USER_MODEL +logger = DaisyLogger(__name__) + + class Dataset(CoreTrackedModel, NotifyMixin, metaclass=CoreNotifyMeta): class Meta: app_label = "core" @@ -291,6 +295,8 @@ def notify(user: "User", obj: "DataDeclaration", verb: "NotificationVerb"): ) on = obj.end_of_storage_duration + logger.info(f"Creating a notification for {user} : {msg}") + Notification.objects.create( recipient=user, verb=verb, diff --git a/core/models/document.py b/core/models/document.py index 09469752..33674369 100644 --- a/core/models/document.py +++ b/core/models/document.py @@ -16,13 +16,17 @@ from django.core.files.storage import default_storage from .utils import CoreModel, CoreNotifyMeta -from notification.models import Notification, NotificationVerb, NotificationSetting +from core.utils import DaisyLogger +from notification.models import Notification, NotificationVerb from notification import NotifyMixin if typing.TYPE_CHECKING: User = settings.AUTH_USER_MODEL +logger = DaisyLogger(__name__) + + def get_file_name(instance, filename): """ Return the path of the final path of the document on the filsystem. @@ -128,6 +132,8 @@ def notify(user: "User", obj: "Document", verb: "NotificationVerb"): msg = f"The Document {obj.shortname} is expiring in {offset} days." on = obj.expiry_date + logger.info(f"Creating a notification for {user} : {msg}") + Notification.objects.create( recipient=user, verb=verb, diff --git a/core/models/project.py b/core/models/project.py index a352914a..5c6f9503 100644 --- a/core/models/project.py +++ b/core/models/project.py @@ -15,7 +15,8 @@ from core import constants from core.permissions.mapping import PERMISSION_MAPPING from notification import NotifyMixin -from notification.models import NotificationVerb, Notification, NotificationSetting +from notification.models import NotificationVerb, Notification +from core.utils import DaisyLogger from .utils import CoreTrackedModel, COMPANY, CoreNotifyMeta from .partner import HomeOrganisation @@ -25,6 +26,9 @@ User = settings.AUTH_USER_MODEL +logger = DaisyLogger(__name__) + + class Project(CoreTrackedModel, NotifyMixin, metaclass=CoreNotifyMeta): class Meta: app_label = "core" @@ -354,6 +358,8 @@ def notify(user: "User", obj: "Project", verb: "NotificationVerb"): msg = f"The project {obj.title} is ending in {offset} days." on = obj.end_date + logger.info(f"Creating a notification for {user} : {msg}") + Notification.objects.create( recipient=user, verb=verb, diff --git a/notification/tasks.py b/notification/tasks.py index e452a815..f4eef1e1 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -25,4 +25,5 @@ def create_notifications_for_entities(execution_date: str = None): logger.info(f"Creating notifications for {exec_date}") for cls in NotifyMixin.__subclasses__(): + logger.info(f"Creating notifications for the {cls} entity...") cls.make_notifications(exec_date) From 2e204b151ee2ec82b783b99b8360c139761ab53a Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 13 Nov 2023 11:08:49 +0100 Subject: [PATCH 36/38] remove excessive list call --- core/models/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/models/dataset.py b/core/models/dataset.py index b7ed550a..908ee422 100644 --- a/core/models/dataset.py +++ b/core/models/dataset.py @@ -261,7 +261,7 @@ def make_notifications_for_user( # Considering users that are indirectly responsible for the dataset (through projects) possible_datasets = set(user.datasets.all()) for project in user.project_set.all(): - possible_datasets.update(list(project.datasets.all())) + possible_datasets.update(project.datasets.all()) for dataset in possible_datasets: # Data Declaration (Embargo Date & End of Storage Duration) for data_declaration in dataset.data_declarations.all(): From e65d977f24a66fe3b4d2dd710d0660e452281c84 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 13 Nov 2023 11:17:04 +0100 Subject: [PATCH 37/38] update the setting template to add schedule of the new notif tast --- elixir_daisy/settings_local.template.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elixir_daisy/settings_local.template.py b/elixir_daisy/settings_local.template.py index 73b1754e..09aa44c9 100644 --- a/elixir_daisy/settings_local.template.py +++ b/elixir_daisy/settings_local.template.py @@ -84,5 +84,9 @@ "clean-accesses-every-day": { "task": "core.tasks.check_accesses_expiration", "schedule": crontab(minute=0, hour=0), # Execute task at midnight - } + }, + "create-notifications-every-day": { + "task": "notification.tasks.create_notifications_for_entities", + "schedule": crontab(minute=15, hour=0), + }, } From 6233822ed806c92828084f6b203d67f6f9d7fc44 Mon Sep 17 00:00:00 2001 From: Hesam KORKI Date: Mon, 13 Nov 2023 11:55:22 +0100 Subject: [PATCH 38/38] run black locally --- elixir_daisy/settings_local.template.py | 2 +- notification/tasks.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/elixir_daisy/settings_local.template.py b/elixir_daisy/settings_local.template.py index 2d99b338..bab151a3 100644 --- a/elixir_daisy/settings_local.template.py +++ b/elixir_daisy/settings_local.template.py @@ -93,7 +93,7 @@ }, "create-notifications-every-day": { "task": "notification.tasks.create_notifications_for_entities", - "schedule": crontab(minute=15, hour=0) + "schedule": crontab(minute=15, hour=0), }, "notifications-email-every-day": { "task": "notification.tasks.send_notifications_for_user_upcoming_events", diff --git a/notification/tasks.py b/notification/tasks.py index a4ccda78..dfdc3930 100644 --- a/notification/tasks.py +++ b/notification/tasks.py @@ -13,6 +13,7 @@ logger = get_task_logger(__name__) + @shared_task def create_notifications_for_entities(execution_date: str = None): """ @@ -26,7 +27,7 @@ def create_notifications_for_entities(execution_date: str = None): exec_date = datetime.now().date() else: exec_date = datetime.strptime(execution_date, "%Y-%m-%d").date() - + logger.info(f"Creating notifications for {exec_date}") for cls in NotifyMixin.__subclasses__():