294 add arbitrary metadata

core/importer/datasets_importer.py

@@ -63,6 +63,8 @@ def get_dataset(elu_accession, title):
         if shares:
             dataset.shares.set(shares, bulk=False)
 
+        dataset.scientific_metadata = dataset_dict.get('metadata', '{}') or '{}'
+
         dataset.save()
         dataset.updated = True
         for local_custodian in local_custodians:
@@ -477,6 +479,7 @@ def _process_study(study):
             has_ethics_approval = study.get('has_ethics_approval', False)
             ethics_approval_notes = study.get('ethics_approval_notes', '')
             url = study.get('url', '')
+            metadata = study.get('metadata', '{}') or '{}'
 
             try:
                 cohort = Cohort.objects.get(title=name)
@@ -493,6 +496,7 @@ def _process_study(study):
             cohort.ethics_confirmation = has_ethics_approval
             cohort.ethics_notes = ethics_approval_notes
             cohort.cohort_web_page = url
+            cohort.scientific_metadata = metadata
             cohort.save()
             cohort.updated = True
 

core/importer/partners_importer.py

@@ -39,6 +39,7 @@ def get_partner(elu_accession, name):
         partner.elu_accession = partner_dict['external_id']
         partner.is_clinical = partner_dict['is_clinical']
         partner.geo_category = partner_dict['geo_category']
+        partner.scientific_metadata = partner_dict.get('metadata', '{}') or '{}'
         partner.sector_category = self.process_sector_category(partner_dict)
         partner.address = partner_dict.get('address') if partner_dict.get('address') else ''
         partner.country_code = partner_dict['country_code']

core/importer/projects_importer.py

@@ -47,6 +47,7 @@ def get_project(elu_accession, title, acronym):
         has_erp = project_dict.get('has_institutional_ethics_approval', False)
         cner_notes = project_dict.get('national_ethics_approval_notes', None)
         erp_notes = project_dict.get('institutional_ethics_approval_notes', None)
+        metadata = project_dict.get('metadata', '{}') or '{}'
 
         try:
             project = get_project(elu_accession=elu_accession,
@@ -82,6 +83,7 @@ def get_project(elu_accession, title, acronym):
         self._process_date_attribute(project, project_dict, "start_date")
         self._process_date_attribute(project, project_dict, "end_date")
 
+        project.scientific_metadata = metadata
         project.save()
 
         local_custodians, local_personnel, external_contacts = self.process_contacts(project_dict.get('contacts', []))

core/migrations/0024_auto_20211004_1610.py

@@ -0,0 +1,34 @@
+# Generated by Django 2.2.24 on 2021-10-04 14:10
+
+import core.models.utils
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0023_remove_datadec_title_unq_constraint'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='cohort',
+            name='scientific_metadata',
+            field=models.TextField(blank=True, default='{}', null=True, validators=[core.models.utils.validate_json], verbose_name='Additional scientific metadata (in JSON format)'),
+        ),
+        migrations.AddField(
+            model_name='dataset',
+            name='scientific_metadata',
+            field=models.TextField(blank=True, default='{}', null=True, validators=[core.models.utils.validate_json], verbose_name='Additional scientific metadata (in JSON format)'),
+        ),
+        migrations.AddField(
+            model_name='partner',
+            name='scientific_metadata',
+            field=models.TextField(blank=True, default='{}', null=True, validators=[core.models.utils.validate_json], verbose_name='Additional scientific metadata (in JSON format)'),
+        ),
+        migrations.AddField(
+            model_name='project',
+            name='scientific_metadata',
+            field=models.TextField(blank=True, default='{}', null=True, validators=[core.models.utils.validate_json], verbose_name='Additional scientific metadata (in JSON format)'),
+        ),
+    ]

core/models/cohort.py

@@ -66,6 +66,7 @@ def to_dict(self):
             'institutes': [i.id for i in self.institutes.all()],
             'has_ethics_approval': self.ethics_confirmation,
             'ethics_approval_notes': self.ethics_notes,
+            "metadata": self.scientific_metadata
         }
         return base_dict
 

core/models/dataset.py

@@ -143,7 +143,8 @@ def to_dict(self):
             "legal_bases": [x.to_dict() for x in self.legal_basis_definitions.all()],
             "storages": storage_dicts,
             "transfers": transfer_dicts,
-            "contacts": contact_dicts
+            "contacts": contact_dicts,
+            "metadata": self.scientific_metadata
         }
         return base_dict
 

core/models/partner.py

@@ -102,7 +102,8 @@ def to_dict(self):
             "geo_category": self.geo_category,
             "sector_category": self.sector_category,
             "address": self.address if self.address else None,
-            "country_code": self.country.ioc_code if self.country.ioc_code else None
+            "country_code": self.country.ioc_code if self.country.ioc_code else None,
+            "metadata": self.scientific_metadata
         }
         return base_dict
 

core/models/project.py

@@ -205,6 +205,7 @@ def to_dict(self):
             "end_date": self.end_date.strftime('%Y-%m-%d') if self.end_date else None,
             "contacts": contact_dicts,
             "publications": pub_dicts,
+            "metadata": self.scientific_metadata
         }
 
         return base_dict

core/models/utils.py

@@ -1,12 +1,29 @@
+from json import loads
+from json.decoder import JSONDecodeError
+
 from django import forms
 from django.conf import settings
+from django.core.exceptions import ValidationError
 from django.db import models
 from django.db.models import TextField
 from django.utils.module_loading import import_string
 
 
 COMPANY = getattr(settings, "COMPANY", 'Company')
 
+def validate_json(value):
+    if len(value) == 0:
+        return value
+
+    try:
+        loads(value)
+        if '{' not in value:  # Very inaccurate, but should do the trick when the user tries to save e.g. '123'
+            raise ValidationError(f'`scientific_metadata` field must be a valid JSON containing a dictionary!')    
+        return value
+    except JSONDecodeError as ex:
+        msg = str(ex)
+        raise ValidationError(f'`scientific_metadata` field must contain a valid JSON! ({msg})')
+
 class classproperty(property):
     def __get__(self, cls, owner):
         return self.fget.__get__(None, owner)()
@@ -21,14 +38,24 @@ class Meta:
 
 
 class CoreTrackedModel(CoreModel):
-    is_published = models.BooleanField(default=False,
-                                       blank=False,
-                                       verbose_name='Is published?')
-    elu_accession = models.CharField(unique=True,
-                                     blank=True,
-                                     null=True,
-                                     max_length=20)
+    elu_accession = models.CharField(
+        unique=True,
+        blank=True,
+        null=True,
+        max_length=20)
+
+    is_published = models.BooleanField(
+        default=False,
+        blank=False,
+        verbose_name='Is published?')
 
+    scientific_metadata = models.TextField(
+        default='{}',
+        blank=True,
+        null=True,
+        verbose_name='Additional scientific metadata (in JSON format)',
+        validators=[validate_json]  # This will work in ModelForm only
+    )
     class Meta:
         abstract = True
 
@@ -42,6 +69,15 @@ def publish(self, save=True):
         if save:
             self.save(update_fields=['is_published', 'elu_accession'])
 
+    def clean(self):
+        cleaned_data = super().clean()
+        validate_json(self.scientific_metadata)
+        return cleaned_data
+
+    def save(self, *args, **kwargs):
+        self.clean()  # Ensure the validator on metadata field is triggered
+        super().save(*args, **kwargs)
+
 
 class TextFieldWithInputWidget(TextField):
 

core/tests/models/test_coretrackedmodel.py

@@ -0,0 +1,28 @@
+from django.core.exceptions import ValidationError
+from pytest import raises
+
+from core.models import Project
+from core.models.utils import validate_json
+from test.factories import ProjectFactory
+
+
+def test_json_validator():
+    with raises(ValidationError):
+        validate_json('{"this is not a valid json; it lacks the value"}')
+
+    validate_json('{"is_it_a_valid_json": "yes, it definitely is!"}')
+    assert True
+
+
+def test_json_vaildation():
+    project = ProjectFactory.create(title='Test project', acronym='Teste projecte')
+    project.save()
+
+    with raises(ValidationError):
+        project.scientific_metadata = '{"this is not a valid json; it lacks the value"}'
+        project.save()
+
+    project.scientific_metadata = '{"is_it_a_valid_json": "yes, it definitely is!"}'
+    project.save()
+    assert True
+