Django 2 Upgrade (part 4). Upgrade to Django 2.2 LTS

dependencies/pip/dev_requirements.in

@@ -6,3 +6,4 @@ ipython
 pytest-django
 pytest-env
 pytest
+mock

dependencies/pip/dev_requirements.txt

@@ -6,6 +6,8 @@
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest
 -e git+https://github.com/jnm/django-haystack@3a3b04564e189c11fdc20a4be43e360b5396708c#egg=django_haystack
+-e git+https://github.com/st8st8/django-markitup@1370b43dc519b1441510a17330b8e980d29a7bf3#egg=django_markitup
+-e git+https://github.com/kobotoolbox/django-request-cache@c240abdd660cc59f5f5280beee30b3e011332a34#egg=django_request_cache
 -e git+https://github.com/kobotoolbox/formpack.git@a08b7626507c2ceff35b816f1e1e0845c85cce26#egg=formpack
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest
 amqp==2.5.2
@@ -35,28 +37,25 @@ django-amazon-ses==2.1.1
 django-braces==1.13.0
 django-celery-beat==1.5.0
 django-constance[database]==2.4.0
-django-cors-headers==2.5.3
+django-cors-headers==3.1.1
 django-debug-toolbar==2.1
 django-extensions==2.2.5
 django-js-asset==1.2.2    # via django-mptt
-django-jsonbfield==0.1.0
 django-loginas==0.3.6
 django-markdownx==2.0.28
-django-markitup==3.0.0
 django-mptt==0.10.0
-django-oauth-toolkit==1.1.0
+django-oauth-toolkit==1.2.0
 django-picklefield==2.0   # via django-constance
 django-private-storage==2.2.1
 django-redis-sessions==0.6.1
 django-registration-redux==2.6
-django-request-cache==1.1
 django-reversion==3.0.1
 django-storages==1.7.2
 django-taggit==1.1.0
 django-timezone-field==3.1  # via django-celery-beat
-django-userforeignkey==0.3.0  # via django-request-cache
+django-userforeignkey==0.3.0
 django-webpack-loader==0.6.0
-django==1.11.26
+django==2.2.7
 djangorestframework-xml==1.4.0
 djangorestframework==3.10.3
 docutils==0.15.2          # via botocore, statistics
@@ -91,7 +90,7 @@ pickleshare==0.7.5        # via ipython
 pillow==6.2.1             # via django-markdownx
 pluggy==0.13.0            # via pytest
 prompt-toolkit==2.0.10    # via ipython
-psycopg2==2.8.4           # via django-jsonbfield
+psycopg2==2.8.4
 ptyprocess==0.6.0         # via pexpect
 py==1.8.0                 # via pytest
 pyasn1==0.4.7
@@ -115,7 +114,7 @@ requests==2.22.0
 responses==0.10.6
 s3transfer==0.2.1         # via boto3
 shortuuid==0.5.0
-six==1.13.0
+six==1.13.0               # via bcrypt, cryptography, django-extensions, jsonschema, mock, packaging, prompt-toolkit, pynacl, pyopenssl, pyrsistent, python-dateutil, responses, traitlets, unittest2
 sqlparse==0.3.0
 static3==0.7.0
 statistics==1.0.3.5

dependencies/pip/external_services.txt

@@ -6,6 +6,8 @@
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest
 -e git+https://github.com/jnm/django-haystack@3a3b04564e189c11fdc20a4be43e360b5396708c#egg=django_haystack
+-e git+https://github.com/st8st8/django-markitup@1370b43dc519b1441510a17330b8e980d29a7bf3#egg=django_markitup
+-e git+https://github.com/kobotoolbox/django-request-cache@c240abdd660cc59f5f5280beee30b3e011332a34#egg=django_request_cache
 -e git+https://github.com/kobotoolbox/formpack.git@a08b7626507c2ceff35b816f1e1e0845c85cce26#egg=formpack
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest
 amqp==2.5.2
@@ -31,28 +33,25 @@ django-amazon-ses==2.1.1
 django-braces==1.13.0
 django-celery-beat==1.5.0
 django-constance[database]==2.4.0
-django-cors-headers==2.5.3
+django-cors-headers==3.1.1
 django-debug-toolbar==2.1
 django-extensions==2.2.5
 django-js-asset==1.2.2    # via django-mptt
-django-jsonbfield==0.1.0
 django-loginas==0.3.6
 django-markdownx==2.0.28
-django-markitup==3.0.0
 django-mptt==0.10.0
-django-oauth-toolkit==1.1.0
+django-oauth-toolkit==1.2.0
 django-picklefield==2.0   # via django-constance
 django-private-storage==2.2.1
 django-redis-sessions==0.6.1
 django-registration-redux==2.6
-django-request-cache==1.1
 django-reversion==3.0.1
 django-storages==1.7.2
 django-taggit==1.1.0
 django-timezone-field==3.1  # via django-celery-beat
-django-userforeignkey==0.3.0  # via django-request-cache
+django-userforeignkey==0.3.0
 django-webpack-loader==0.6.0
-django==1.11.26
+django==2.2.7
 djangorestframework-xml==1.4.0
 djangorestframework==3.10.3
 docutils==0.15.2          # via botocore, statistics
@@ -69,13 +68,12 @@ kombu==4.6.6
 linecache2==1.0.0         # via traceback2
 lxml==4.4.1
 markdown==3.1.1
-mock==3.0.5
 more-itertools==7.2.0     # via zipp
 ndg-httpsclient==0.5.1
 oauthlib==3.1.0
 path.py==12.0.2
 pillow==6.2.1             # via django-markdownx
-psycopg2==2.8.4           # via django-jsonbfield
+psycopg2==2.8.4
 pyasn1==0.4.7
 pycparser==2.19           # via cffi
 pygments==2.4.2
@@ -93,7 +91,7 @@ requests==2.22.0
 responses==0.10.6
 s3transfer==0.2.1         # via boto3
 shortuuid==0.5.0
-six==1.13.0
+six==1.13.0               # via cryptography, django-extensions, jsonschema, pyopenssl, pyrsistent, python-dateutil, responses, transifex-client, unittest2
 sqlparse==0.3.0
 static3==0.7.0
 statistics==1.0.3.5

dependencies/pip/requirements.in

@@ -1,7 +1,7 @@
 # File for use with `pip-compile`; see https://github.com/nvie/pip-tools
 # https://github.com/bndr/pipreqs is a handy utility, too.
 
-# Formpack
+# FormPack
 -e git+https://github.com/kobotoolbox/formpack.git@a08b7626507c2ceff35b816f1e1e0845c85cce26#egg=formpack
 
 # More up-to-date version of django-digest than PyPI seems to have.
@@ -12,8 +12,14 @@
 # TODO: Get Haystack from PyPI again once https://github.com/django-haystack/django-haystack/issues/1662 is fixed
 -e git+https://github.com/jnm/django-haystack@3a3b04564e189c11fdc20a4be43e360b5396708c#egg=django_haystack
 
+# TODO²: Get `django-markitup` from PyPI again once it supports Django 2.1
+-e git+https://github.com/st8st8/django-markitup@1370b43dc519b1441510a17330b8e980d29a7bf3#egg=django_markitup
+
+# TODO³: Get `django-request-cache` from PyPI again once https://github.com/anexia-it/django-request-cache/pull/6 is merged
+-e git+https://github.com/kobotoolbox/django-request-cache@c240abdd660cc59f5f5280beee30b3e011332a34#egg=django_request_cache
+
 # Regular PyPI packages
-Django<2
+Django>=2.2,<2.3
 Markdown
 Pygments
 amqp
@@ -28,17 +34,16 @@ dj-database-url
 django-braces
 django-celery-beat
 django-constance[database]
-django-cors-headers<3 # 3.0.0 dropped support for Python2 support
+django-cors-headers
 django-debug-toolbar
 django-extensions
-django-jsonbfield
-django-oauth-toolkit<1.1.1 # Last Python2 support: 1.1.0
+django-oauth-toolkit
 django-registration-redux
 django-amazon-ses
 django-webpack-loader
 django-loginas
 django-markdownx
-django-markitup
+# django-markitup # Not compatible with Django 2.0+ (see above ToDo²)
 django-mptt
 django-reversion<3.0.2 # Migration issue with 3.0.2
 django-taggit
@@ -53,17 +58,16 @@ geojson-rewind
 jsonfield
 kombu
 lxml
-mock
 oauthlib
 #py-gfm # Incompatible with markdown 3.x
+psycopg2
 pymongo
 python-dateutil
 pytz
 pyxform
 requests
 responses
 shortuuid
-six
 sqlparse
 static3
 tabulate
@@ -79,4 +83,4 @@ ndg-httpsclient
 pyasn1
 
 # Cache-Request
-django-request-cache
+# django-request-cache # Not compatible with Django 2.1+ (see above ToDo³)

dependencies/pip/requirements.txt

@@ -6,6 +6,8 @@
 #
 -e git+https://github.com/dimagi/django-digest@52ba7edeb326efd97d5670273bb6fa8b0539e501#egg=django_digest
 -e git+https://github.com/jnm/django-haystack@3a3b04564e189c11fdc20a4be43e360b5396708c#egg=django_haystack
+-e git+https://github.com/st8st8/django-markitup@1370b43dc519b1441510a17330b8e980d29a7bf3#egg=django_markitup
+-e git+https://github.com/kobotoolbox/django-request-cache@c240abdd660cc59f5f5280beee30b3e011332a34#egg=django_request_cache
 -e git+https://github.com/kobotoolbox/formpack.git@a08b7626507c2ceff35b816f1e1e0845c85cce26#egg=formpack
 -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest
 amqp==2.5.2
@@ -31,28 +33,25 @@ django-amazon-ses==2.1.1
 django-braces==1.13.0
 django-celery-beat==1.5.0
 django-constance[database]==2.4.0
-django-cors-headers==2.5.3
+django-cors-headers==3.1.1
 django-debug-toolbar==2.1
 django-extensions==2.2.5
 django-js-asset==1.2.2    # via django-mptt
-django-jsonbfield==0.1.0
 django-loginas==0.3.6
 django-markdownx==2.0.28
-django-markitup==3.0.0
 django-mptt==0.10.0
-django-oauth-toolkit==1.1.0
+django-oauth-toolkit==1.2.0
 django-picklefield==2.0   # via django-constance
 django-private-storage==2.2.1
 django-redis-sessions==0.6.1
 django-registration-redux==2.6
-django-request-cache==1.1
 django-reversion==3.0.1
 django-storages==1.7.2
 django-taggit==1.1.0
 django-timezone-field==3.1  # via django-celery-beat
-django-userforeignkey==0.3.0  # via django-request-cache
+django-userforeignkey==0.3.0
 django-webpack-loader==0.6.0
-django==1.11.26
+django==2.2.7
 djangorestframework-xml==1.4.0
 djangorestframework==3.10.3
 docutils==0.15.2          # via botocore, statistics
@@ -69,13 +68,12 @@ kombu==4.6.6
 linecache2==1.0.0         # via traceback2
 lxml==4.4.1
 markdown==3.1.1
-mock==3.0.5
 more-itertools==7.2.0     # via zipp
 ndg-httpsclient==0.5.1
 oauthlib==3.1.0
 path.py==12.0.2
 pillow==6.2.1             # via django-markdownx
-psycopg2==2.8.4           # via django-jsonbfield
+psycopg2==2.8.4
 pyasn1==0.4.7
 pycparser==2.19           # via cffi
 pygments==2.4.2
@@ -92,7 +90,7 @@ requests==2.22.0
 responses==0.10.6
 s3transfer==0.2.1         # via boto3
 shortuuid==0.5.0
-six==1.13.0
+six==1.13.0               # via cryptography, django-extensions, jsonschema, pyopenssl, pyrsistent, python-dateutil, responses, unittest2
 sqlparse==0.3.0
 static3==0.7.0
 statistics==1.0.3.5

hub/migrations/0005_perusersetting.py

@@ -1,5 +1,5 @@
 # coding: utf-8
-import jsonbfield.fields
+from django.contrib.postgres.fields import JSONField as JSONBField
 from django.db import migrations, models
 
 
@@ -14,7 +14,7 @@ class Migration(migrations.Migration):
             name='PerUserSetting',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('user_queries', jsonbfield.fields.JSONField(help_text='A JSON representation of a *list* of Django queries, e.g. `[{"email__endswith": "@kobotoolbox.org"}, {"email__endswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.')),
+                ('user_queries', JSONBField(help_text='A JSON representation of a *list* of Django queries, e.g. `[{"email__endswith": "@kobotoolbox.org"}, {"email__endswith": "@kbtdev.org"}]`. A matching user is one who would be returned by ANY of the queries in the list.')),
                 ('name', models.CharField(unique=True, max_length=255)),
                 ('value_when_matched', models.CharField(max_length=2048, blank=True)),
                 ('value_when_not_matched', models.CharField(max_length=2048, blank=True)),

hub/migrations/0006_remove_formbuilder_preference_table.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.25 on 2019-10-17 17:22
-from django.db import migrations, models
-import jsonbfield.fields
+from django.db import migrations
 
 
 class Migration(migrations.Migration):

hub/models.py

@@ -1,14 +1,14 @@
 # coding: utf-8
 from django.conf import settings
 from django.contrib.auth.models import User
+from django.contrib.postgres.fields import JSONField as JSONBField
 from django.core.exceptions import FieldError, ValidationError
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.db import models
 from django.db.models.signals import post_save
 from django.http import HttpResponseRedirect
 from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext_lazy as _
-from jsonbfield.fields import JSONField as JSONBField
 from jsonfield import JSONField
 from markitup.fields import MarkupField
 

hub/tests/test_perusersetting.py

@@ -1,6 +1,6 @@
 # coding: utf-8
 from django.contrib.auth.models import User, AnonymousUser
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.test import TestCase
 
 from hub.models import PerUserSetting

kobo/apps/external_integrations/__init__.py

@@ -6,3 +6,8 @@
 class ExternalIntegrationsAppConfig(AppConfig):
     name = 'kobo.apps.external_integrations'
     verbose_name = _('External integrations')
+
+    def ready(self):
+        # Makes sure all signal handlers are connected
+        from kobo.apps.external_integrations import signals  # noqa
+        super().ready()

kobo/apps/external_integrations/models.py

@@ -1,27 +1,23 @@
 # coding: utf-8
-from corsheaders.models import AbstractCorsModel
+from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
 
-def _set_cors_field_options(name, bases, attrs):
-    cls = type(name, bases, attrs)
-    # The `cors` field is already defined by `AbstractCorsModel`, but let's
-    # help folks out by giving it a more descriptive name and help text, which
-    # will both appear in the admin interface
-    cors_field = cls._meta.get_field('cors')
-    cors_field.verbose_name = _('allowed origin')
-    cors_field.help_text = _('do not include http:// or https://')
-    return cls
-
-
-class CorsModel(AbstractCorsModel, metaclass=_set_cors_field_options):
+class CorsModel(models.Model):
     """
     A model with one field, `cors`, which specifies an allowed origin that must
-    exactly match the `netloc` returned by `urlparse`
+    exactly match `request.META.get('HTTP_ORIGIN')`
+    """
 
-    ToDo: `AbstractCorsModel` has been removed in v3.0.0.
-    Update code to support `django-cors-headers` v3.x
-   """
+    cors = models.CharField(
+        max_length=255,
+        verbose_name=_('allowed origin'),
+        help_text=_(
+            'Must contain exactly the URI scheme, host, and port, e.g. '
+            'https://example.com:1234. Standard ports (80 for http and 443 '
+            'for https) may be omitted.'
+        )
+    )
 
     def __str__(self):
         return self.cors

kobo/apps/external_integrations/signals.py

@@ -0,0 +1,12 @@
+# coding: utf-8
+from corsheaders.signals import check_request_enabled
+
+from .models import CorsModel
+
+
+def cors_allow_external_sites(sender, request, **kwargs):
+    origin = request.META.get('HTTP_ORIGIN')
+    return CorsModel.objects.filter(cors=origin).exists()
+
+
+check_request_enabled.connect(cors_allow_external_sites)

kobo/apps/external_integrations/tests/test_cors.py

@@ -1,6 +1,6 @@
 # coding: utf-8
+from django.urls import reverse
 from rest_framework.test import APITestCase
-from django.core.urlresolvers import reverse
 
 from ..models import CorsModel
 
@@ -25,8 +25,7 @@ def test_no_cors_response_with_untrusted_origin(self):
 
     def test_cors_response_with_trusted_origin(self):
         trusted_origin = 'https://www.fsf.org'
-        trusted_netloc = 'www.fsf.org'
-        CorsModel.objects.create(cors=trusted_netloc)
+        CorsModel.objects.create(cors=trusted_origin)
         response = self.client.get(
             self.innocuous_url,
             HTTP_ORIGIN=trusted_origin,