merge develop to fixes-to-importer

DEPLOYMENT.md

@@ -8,7 +8,7 @@ sudo yum install python36-devel openldap-devel nginx
 sudo yum group install "Development Tools"
 
 wget https://bootstrap.pypa.io/get-pip.py
-sudo python36 get-pip.py
+sudo python3.6 get-pip.py
 ```
 
 
@@ -22,7 +22,7 @@ sudo useradd daisy
 sudo usermod -a -G users daisy
 sudo su - daisy
 mkdir config log
-git clone git@github.com:elixir-luxembourg/daisy.git
+git clone https://github.com/elixir-luxembourg/daisy.git
 exit
 sudo /usr/local/bin/pip install -e /home/daisy/daisy
 sudo /usr/local/bin/pip install gunicorn
@@ -69,6 +69,10 @@ You need configure the solr core 'daisy'. To do so you need to create 'schema.xm
 ```bash
 sudo cp /home/daisy/daisy/docker/solr/schema.xml /var/solr/data/daisy/conf/
 sudo cp /home/daisy/daisy/docker/solr/solrconfig.xml /var/solr/data/daisy/conf/
+```
+
+Grant ownership and change privileges of `/var/solr` folder
+```
 sudo chown -R solr:users /var/solr
 sudo chmod -R 775 /var/solr
 ```
@@ -246,8 +250,6 @@ sudo systemctl start celery_beat
 
 ### Install database server
 
-Documentation from: https://www.postgresql.org/download/linux/redhat/
-
 ```bash
 sudo yum install https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-7-x86_64/pgdg-centos10-10-2.noarch.rpm
 sudo yum install postgresql10
@@ -257,6 +259,8 @@ sudo systemctl enable postgresql-10
 sudo systemctl start postgresql-10
 ```
 
+In case the installation fails, follow steps in the [official documentation](https://www.postgresql.org/download/linux/redhat/) for installation of Postgresql 10 on your platform.
+
 ### Create database and roles
 
 
@@ -306,6 +310,13 @@ cp /home/daisy/daisy/elixir_daisy/settings_local.template.py   /home/daisy/daisy
 vi /home/daisy/daisy/elixir_daisy/settings_local.py
 ```
 
+<span style="color:red;">Change SECRET_KEY variable:</span>
+
+```
+# SECURITY WARNING: change the secret key used in production and keep it secret !
+SECRET_KEY='<your-new-secret-key>'
+```
+
 Put in the following database configuration to the 'settings_local.py' file.
 
 ```
@@ -452,18 +463,19 @@ To do this run the following.
 ```bash
 sudo su - daisy
 cd /home/daisy/daisy
-python36 manage.py collectstatic 
-python36 manage.py migrate 
-python36 manage.py build_solr_schema -c /var/solr/data/daisy/conf -r daisy  
+python3.6 manage.py collectstatic 
+python3.6 manage.py migrate 
+python3.6 manage.py build_solr_schema -c /var/solr/data/daisy/conf -r daisy  
 cd /home/daisy/daisy/core/fixtures/
 wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/edda.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hpo.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hdo.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hgnc.json
-python36 manage.py load_initial_data
+cd /home/daisy/daisy
+python3.6 manage.py load_initial_data
 ```
 The load_initial_data command needs several minutes to complete.
 DAISY has a demo data loader. With example records of Projects Datasets and Users. If you want to deploy DAISY  demo data, then do 
 
 ```bash
-python36 manage.py load_demo_data
+python3.6 manage.py load_demo_data
 ```
 
 The above command will create an 'admin' and other users such as 'alice.white', 'john.doe' 'jane.doe'. The password for all  is 'demo'.
@@ -472,13 +484,13 @@ The above command will create an 'admin' and other users such as 'alice.white',
 If you do not want to load the demo data and work with your own definitions, then you'd still need to create super user for the application, with which you can logon and create other users as well as records. To create a super user, do the following and respond to the questions. 
 
 ```bash
-python36 manage.py createsuperuser
+python3.6 manage.py createsuperuser
 ```
 
 Trigger a reindex with:
 
 ```bash
-python36 manage.py rebuild_index
+python3.6 manage.py rebuild_index
 ```
 
 # Validate the installation
@@ -559,7 +571,7 @@ As daisy user:
 
 ```bash
 cd /home/daisy/daisy
-python36 manage.py migrate && python36 manage.py build_solr_schema -c /var/solr/data/daisy/conf/ -r daisy && yes | python36 manage.py clear_index && yes "yes" | python36 manage.py collectstatic;
+python3.6 manage.py migrate && python3.6 manage.py build_solr_schema -c /var/solr/data/daisy/conf/ -r daisy && yes | python3.6 manage.py clear_index && yes "yes" | python3.6 manage.py collectstatic;
 ```
 
 
@@ -575,7 +587,7 @@ cd /home/daisy/daisy/core/fixtures/
 wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/edda.json -O edda.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hpo.json -O hpo.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hdo.json -O hdo.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hgnc.json -O hgnc.json
 
 cd /home/daisy/daisy
-python36 manage.py load_initial_data
+python3.6 manage.py load_initial_data
 ```     
   
 **IMPORTANT NOTE:** This step can take several minutes to complete. 
@@ -586,15 +598,15 @@ python36 manage.py load_initial_data
 If LDAP was used to import users, they have to be imported again.
 As daisy user:
 ```bash
-python36 manage.py import_users
+python3.6 manage.py import_users
 ```
 
 6) Rebuild Solr search index.
 
 As daisy user:
  ```bash
  cd /home/daisy/daisy
- python36 manage.py rebuild_index
+ python3.6 manage.py rebuild_index
 ```
 
 7) Restart services.

elixir_daisy/settings_local.template.py

@@ -8,9 +8,28 @@
     'guardian.backends.ObjectPermissionBackend',
 ]
 
+
+#SECURITY WARNING: change the secret key used in production and keep it secret !
+SECRET_KEY = None  # Generate a secret key by e.g. django.core.management.utils.get_random_secret_key()
+if SECRET_KEY is None: raise NotImplementedError('You must specify SECRET_KEY in settings_local.py')
+
+
+
 COMPANY = 'LCSB'  # Used for generating some models' verbose names
 
+# Placeholders on login page
+# LOGIN_USERNAME_PLACEHOLDER = ''
+# LOGIN_PASSWORD_PLACEHOLDER = ''
+
+# Optional username suffixes
+# Setting this variable allows the user to login with a prefix only. Suffix is concatenated to create full user name)
+# LOGIN_USERNAME_SUFFIX = ''
+# This variable will be stripped from an entered user name and replaced by LOGIN_USERNAME_SUFFIX to create full user name
+# LOGIN_USERNAME_ALTERNATIVE_SUFFIX = ''
+
 # Uncomment the following lines if LDAP authentication will be used and user definitions will be bulk imported from LDAP
+# import ldap
+# from django_auth_ldap.config import LDAPSearch, LDAPSearchUnion
 # AUTHENTICATION_BACKENDS = ['django_auth_ldap.backend.LDAPBackend'] + AUTHENTICATION_BACKENDS
 # AUTH_LDAP_SERVER_URI = 'ldap://XXXXXXX:389'
 # AUTH_LDAP_BIND_DN = ""

elixir_daisy/urls.py

@@ -24,7 +24,7 @@
 from web.urls import web_urls
 
 urlpatterns = [
-    url(r'^login/$', auth_views.LoginView.as_view(), name='login', kwargs={"authentication_form": UserAuthForm}),
+    url(r'^login/$', auth_views.LoginView.as_view(authentication_form=UserAuthForm), name='login'),
     url(r'^logout/$', auth_views.LogoutView.as_view(), name='logout'),
     url(r'^admin/', admin.site.urls),
     url(r'', include(web_urls)),

setup.py

@@ -7,12 +7,12 @@
     from distutils.core import setup
 
 requirements = [
-    'Django==2.2.18',
+    'Django==2.2.20',
     'django-auth-ldap==1.7.0',
     'django-celery-beat==1.4.0',
     'django-celery-results==1.0.4',
     'django-compressor==2.2',
-    'django-debug-toolbar==1.11',
+    'django-debug-toolbar==1.11.1',
     'django-enumchoicefield==1.1.0',
     'django-excel-response==2.0.4',
     'xlsxwriter==1.2.9',

web/templates/datasets/dataset.html

@@ -127,7 +127,7 @@ <h2 class="card-title">Data declarations</h2>
                                 <td>{{ data_declaration.cohorts.all | join:', ' }}</td>
                                 <td>{{ data_declaration.partner }}</td>
                                 <td>{%if data_declaration.data_types  %}{{ data_declaration.data_types | join:', ' }} {% else %}<p class="alert-error"> missing </p>{% endif %}</td>
-                                <td>{{ data_declaration.special_subjects | yesno:"yes,no,unknown" }}</td>
+                                <td>{{ data_declaration.has_special_subjects | yesno:"yes,no,unknown" }}</td>
                                 <td>{{ data_declaration.end_of_storage_duration  | default:"" }}</td>
                                 {% if can_edit %}
                                     <td>

web/views/api.py

@@ -1,13 +1,14 @@
 import json
 import os
 
-from functools import wraps
 from io import StringIO
 
 from django.conf import settings
 from django.core.paginator import Paginator
 from django.http import JsonResponse, HttpResponse, HttpResponseBadRequest
 
+from ontobio import obograph_util, Ontology
+
 from stronghold.decorators import public
 
 from core.importer.datasets_exporter import DatasetsExporter
@@ -18,8 +19,6 @@
 
 from elixir_daisy import settings
 
-from ontobio import obograph_util, Ontology
-
 
 logger = DaisyLogger(__name__)
 
@@ -95,25 +94,18 @@ def termsearch(request, category):
         }
     })
 
-def requires_apikey(view_func):
-    """
-    A decorator that checks if the API_KEY is present
-    and the value is valid
-    """
-    def _wrapped_view(request, *args, **kwargs):
-        if 'API_KEY' not in request.GET:
-            return JsonResponse({
-                'status': 'Error',
-                'description': 'API_KEY missing or invalid'
-            }, status=403)
-        elif request.GET.get('API_KEY') != getattr(settings, 'GLOBAL_API_KEY'):
-            return JsonResponse({
-                'status': 'Error',
-                'description': 'API_KEY missing or invalid'
-            }, status=403)
-        return view_func(request, *args, **kwargs)
-
-    return _wrapped_view
+@public
+def datasets(request):
+    if 'API_KEY' not in request.GET:
+        return JsonResponse({
+            'status': 'Error',
+            'description': 'API_KEY missing or invalid'
+        }, status=403)
+    elif request.GET.get('API_KEY') != getattr(settings, 'GLOBAL_API_KEY'):
+        return JsonResponse({
+            'status': 'Error',
+            'description': 'API_KEY missing or invalid'
+        }, status=403)
 
 @public
 @requires_apikey
@@ -135,10 +127,9 @@ def datasets(request):
             'status': 'Error',
             'description': 'Something went wrong during exporting the datasets',
             'more': str(e)
-        })
+        }, status=500)
 
 @public
-@requires_apikey
 def projects(request):
     if 'title' in request.GET:
         title = request.GET.get('title', '')
@@ -157,4 +148,4 @@ def projects(request):
             'status': 'Error',
             'description': 'Something went wrong during exporting the projects',
             'more': str(e)
-        })
+        }, status=500)