Fixes to importer

README.md

@@ -212,7 +212,7 @@ Single file mode:
 
 Batch mode:
 ```bash
-./manage.py import_projects -f path/to/dir/with/json/files/
+./manage.py import_projects -d path/to/dir/with/json/files/
 ```
 
 Available commands: `import_projects`, `import_datasets`, `import_partners`.

core/fixtures/contact-types.json

@@ -1,4 +1,7 @@
 [
+  {
+    "name": "Researcher"
+  },
   {
     "name": "Principal_Investigator"
   },

core/importer/base_importer.py

@@ -149,67 +149,18 @@ def process_contacts(self, contacts_list: List[Dict]):
             first_name = contact_dict.get('first_name').strip()
             last_name = contact_dict.get('last_name').strip()
             email = contact_dict.get('email', '').strip()
-            full_name = f"{first_name} {last_name}"
-            role_name = contact_dict.get('role')
-            _is_local_contact = self.is_local_contact(contact_dict)
-            if _is_local_contact:
-                user = User.objects.filter(first_name__icontains=first_name.lower(),
-                                           last_name__icontains=last_name.lower())
-                if len(user) > 1:
-                    users = User.objects.filter(first_name__icontains=first_name.lower(),
-                                                last_name__icontains=last_name.lower(),
-                                                email=email)
-                    if len(users) != 1:
-                        msg = 'Something went wrong - there are two contacts with the same first and last name, and it''s impossible to differentiate them'
-                        self.logger.warning(msg)
-                    user = users.first()
-                elif len(user) == 1:
-                    user = user.first()
-                else:
-                    user = None
-                if user is None:
-                    self.logger.warning(f'No user found for {full_name} - hence an inactive user will be created')
-
-                    usr_name = first_name.lower() + '.' + last_name.lower()
-                    user = User.objects.create(username=usr_name, 
-                                              password='', 
-                                              first_name=first_name, 
-                                              last_name=last_name, 
-                                              is_active=False,
-                                              email=email)
-                    user.staff = True
-
-                    if role_name == PRINCIPAL_INVESTIGATOR:
-                        g = Group.objects.get(name=GroupConstants.VIP.value)
-                        user.groups.add(g)
-
-                    user.save()
+            role_name = self.validate_contact_type(contact_dict.get('role'))
+            affiliations = contact_dict.get('affiliations', [])
+            if self.is_local_contact(contact_dict):
+                user = self.process_local_contact(first_name, last_name, email, role_name, affiliations)
                 if role_name == PRINCIPAL_INVESTIGATOR:
                     local_custodians.append(user)
                 else:
                     local_personnel.append(user)
 
             else:
-                contact = (Contact.objects.filter(first_name__icontains=first_name.lower(),
-                                                  last_name__icontains=last_name.lower()) | Contact.objects.filter(
-                    first_name__icontains=first_name.upper(), last_name__icontains=last_name.upper())).first()
-                if contact is None:
-                    contact_type_pi, _ = ContactType.objects.get_or_create(name=role_name)
-                    contact, _ = Contact.objects.get_or_create(
-                        first_name=first_name,
-                        last_name=last_name,
-                        email=email,
-                        type=contact_type_pi
-                    )
-                    affiliations = contact_dict.get('affiliations')
-                    for affiliation in affiliations:
-                        partner = Partner.objects.filter(name=affiliation)
-                        if len(partner):
-                            contact.partners.add(partner[0])
-                        else:
-                            self.logger.warning(f'no partner found for the affiliation: {affiliation}')
-                    contact.save()
-                    external_contacts.append(contact)
+                contact = self.process_external_contact(first_name, last_name, email, role_name, affiliations)
+                external_contacts.append(contact)
 
         return local_custodians, local_personnel, external_contacts
 
@@ -237,5 +188,73 @@ def process_date(self, date_string):
     @staticmethod
     def is_local_contact(contact_dict):
         home_organisation = Partner.objects.get(acronym=settings.COMPANY)
-        _is_local_contact = home_organisation.name in contact_dict.get("affiliations")
+        _is_local_contact = home_organisation.name in contact_dict.get("affiliations") or home_organisation.acronym in contact_dict.get("affiliations")
         return _is_local_contact
+
+    def validate_contact_type(self, contact_type):
+        try:
+            contact_type_obj = ContactType.objects.get(name=contact_type)
+        except ContactType.DoesNotExist:
+            self.logger.warning(f'Unknown contact type: {contact_type}. Setting to "Other".')
+            contact_type = 'Other'
+        return contact_type
+
+    def process_local_contact(self, first_name, last_name, email, role_name, affiliations):
+        user = User.objects.filter(first_name__icontains=first_name,last_name__icontains=last_name)
+        if len(user) > 1:
+            users = User.objects.filter(first_name__icontains=first_name,
+                                        last_name__icontains=last_name,
+                                        email=email)
+            if len(users) != 1:
+                msg = 'Something went wrong - there are two contacts with the same first and last name, and it''s impossible to differentiate them'
+                self.logger.warning(msg)
+            user = users.first()
+        elif len(user) == 1:
+            user = user.first()
+        else:
+            user = None
+        if user is None:
+            self.logger.warning(f"No user found for '{first_name} {last_name}' - hence an inactive user will be created")
+
+            usr_name = first_name.lower() + '.' + last_name.lower()
+            user = User.objects.create(username=usr_name,
+                                        password='',
+                                        first_name=first_name,
+                                        last_name=last_name,
+                                        is_active=False,
+                                        email=email)
+            user.staff = True
+
+            if role_name == PRINCIPAL_INVESTIGATOR:
+                g = Group.objects.get(name=GroupConstants.VIP.value)
+                user.groups.add(g)
+            user.save()
+        return user
+
+    def process_external_contact(self, first_name, last_name, email, role_name, affiliations):
+        contact = (
+            Contact.objects.filter(
+                first_name__icontains=first_name,
+                last_name__icontains=last_name,
+                partners__name__in=affiliations) |
+            Contact.objects.filter(
+                first_name__icontains=first_name,
+                last_name__icontains=last_name,
+                partners__acronym__in=affiliations)
+                ).first()
+        if contact is None:
+            contact = Contact.objects.create(
+                first_name=first_name,
+                last_name=last_name,
+                email=email,
+                type=ContactType.objects.get(name=role_name)
+            )
+            for affiliation in affiliations:
+                partner = Partner.objects.filter(name=affiliation)
+                if len(partner):
+                    contact.partners.add(partner[0])
+                else:
+                    self.logger.warning(f"Cannot link contact '{first_name} {last_name}' to partner. No partner found for the affiliation: {affiliation}")
+            contact.save()
+        return contact
+

core/importer/datasets_importer.py

@@ -258,7 +258,7 @@ def process_datadeclaration(self, datadec_dict, dataset):
             datadec = DataDeclaration.objects.create(title=title, dataset=dataset)
 
         if 'source_study' not in datadec_dict or len(datadec_dict.get('source_study')) == 0:
-            self.logger.warning("Data declaration with has no `source_study` set - there will be a problem processing study/cohort data.")
+            self.logger.warning(f"Data declaration with title '{title_to_show}' has no `source_study` set - there will be a problem processing study/cohort data.")
 
         datadec.has_special_subjects = datadec_dict.get('has_special_subjects', False)
         datadec.data_types_notes = datadec_dict.get('data_type_notes', None)
@@ -496,19 +496,31 @@ def _process_study(study):
             description = study.get('description', '')
             has_ethics_approval = study.get('has_ethics_approval', False)
             ethics_approval_notes = study.get('ethics_approval_notes', '')
-            url = study.get('url', '')  # TODO: Currently this is lost
+            url = study.get('url', '')
 
-            cohort, _ = Cohort.objects.get_or_create(
-                ethics_confirmation=has_ethics_approval,
-                comments=description,
-                title=name,
-            )
+            try:
+                cohort = Cohort.objects.get(title=name)
+            except Cohort.DoesNotExist:
+                cohort = None
+
+            if cohort:
+                msg = f"Cohort with title '{safe_name}' already found. All fields are going to be updated."
+                self.logger.warning(msg)
+            else:
+                cohort = Cohort.objects.create(title=name)
+
+            cohort.description = description
+            cohort.ethics_confirmation = has_ethics_approval
+            cohort.ethics_notes = ethics_approval_notes
+            cohort.cohort_web_page = url
             cohort.save()
+            cohort.updated = True
 
             local_custodians, local_personnel, external_contacts = self.process_contacts(study.get("contacts", []))
             cohort.owners.set(external_contacts)
 
             cohort.save()
+            cohort.updated = True
             msg = f"Cohort '{safe_name}' imported successfully. Will try to link it to the data declaration..."
             self.logger.info(msg)
 

core/migrations/0014_cohort_url_ethics_notes.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.13 on 2021-03-24 23:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0013_userestriction_use_class_note'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='cohort',
+            name='cohort_web_page',
+            field=models.URLField(blank=True, help_text='If the cohort has a webpage, please provide its URL link here.', verbose_name='Cohorts URL page'),
+        ),
+        migrations.AddField(
+            model_name='cohort',
+            name='ethics_notes',
+            field=models.TextField(blank=True, default='', help_text='Provide notes on ethics approval. If it does not exist, please state justifications here.', null=True, verbose_name='Ethics Approval notes'),
+        ),
+    ]

core/models/cohort.py

@@ -18,6 +18,16 @@ class AppMeta:
                                                verbose_name='Confirmation of Ethics Approval?',
                                    help_text='Is the existence of the study\'s ethics approval confirmed by the cohort owner.')
 
+    ethics_notes = models.TextField(verbose_name='Ethics Approval notes',
+                                 default='',
+                                 help_text='Provide notes on ethics approval. If it does not exist, please state justifications here.',
+                                 blank=True,
+                                 null=True)
+
+    cohort_web_page = models.URLField(verbose_name='Cohort''s URL page',
+                                       help_text='If the cohort has a webpage, please provide its URL link here.',
+                                       blank=True)
+
     comments = models.TextField(verbose_name='Comments', blank=True, null=True,
                                 help_text='Any additional remarks on this cohort.')
 
@@ -51,7 +61,9 @@ def to_dict(self):
             'owners': owners_dicts,
             'title': self.title,
             'institutes': [i.id for i in self.institutes.all()],
-            'ethics_confirmation': self.ethics_confirmation
+            'ethics_confirmation': self.ethics_confirmation,
+            'ethics_approval_notes': self.ethics_notes,
+            'url': self.cohort_web_page
         }
         return base_dict
 

core/tests/importer/test_datasets_importer.py

@@ -14,7 +14,7 @@ def test_dummy(celery_session_worker, storage_resources, can_defer_constraint_ch
 
 
 @pytest.mark.django_db
-def test_import_datasets(celery_session_worker, storage_resources, data_types, partners, gdpr_roles, can_defer_constraint_checks):
+def test_import_datasets(celery_session_worker, storage_resources, contact_types, data_types, partners, gdpr_roles, can_defer_constraint_checks):
     VIP = factories.VIPGroup()
 
     factories.UserFactory.create(first_name='Igor', last_name='Teal', groups=[VIP], email="user@uni.edu")

core/tests/importer/test_projects_importer.py

@@ -8,7 +8,7 @@
 
 
 @pytest.mark.django_db
-def test_import_projects(celery_session_worker, partners):
+def test_import_projects(celery_session_worker, contact_types, partners):
 
     VIP = factories.VIPGroup()