feat(async): add uvicorn, asgi, upgrade django=4.1 (#473)

> Why was this change necessary?

Current version of django(3.2) doesn't support async tasks.

> How does it address the problem?

After upgrading to django>4.1 and running asgi server, the application
can run tasks asynchronously.

> Are there any side effects?

Uvicorn config `--limit-concurrency` is not yet supported when running
with Gunicorn.

Need to test provisioner scripts before merging

README.md

@@ -12,7 +12,7 @@
 
 ## Features
 
-- Django 3.2.x
+- Django 4.1.x
 - Python 3.9.x
 - [Poetry][poetry] Support
 - Support for [black](https://pypi.org/project/black/)!

cookiecutter-test-config.yaml

@@ -2,3 +2,4 @@ default_context:
     enable_whitenoise: "y"
     add_celery: "y"
     add_graphql: "y"
+    add_asgi: "y"

cookiecutter.json

@@ -19,6 +19,7 @@
     , "add_django_auth_wall": "y"
     , "add_celery": "n"
     , "add_graphql": "n"
+    , "add_asgi": "n"
     , "add_pre_commit": "y"
     , "add_docker": "y"
     , "pagination": ["LimitOffsetPagination", "CursorPagination"]

hooks/post_gen_project.sh

@@ -51,6 +51,12 @@ if echo "{{ cookiecutter.add_graphql }}" | grep -iq "^n"; then
     rm -rf tests/graphql
 fi
 
+if echo "{{ cookiecutter.add_asgi }}" | grep -iq "^n"; then
+    rm -rf asgi.py
+else
+    rm -rf wsgi.py
+fi
+
 if echo "$yn" | grep -iq "^y"; then
     echo "==> Checking system dependencies. You may need to enter your sudo password."
 

{{cookiecutter.github_repository}}/asgi.py

@@ -0,0 +1,15 @@
+# Standard Library
+import os
+
+# Third Party Stuff
+from django.core.asgi import get_asgi_application
+from dotenv import load_dotenv
+
+# Read .env file and set key/value inside it as environment variables
+# see: http://github.com/theskumar/python-dotenv
+load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
+
+# We defer to a DJANGO_SETTINGS_MODULE already in the environment.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.production")
+
+application = get_asgi_application()

{{cookiecutter.github_repository}}/compose/dev/django/start

@@ -6,5 +6,8 @@ set -o nounset
 
 
 python /app/manage.py collectstatic --noinput
-# /usr/local/bin/gunicorn asgi --bind 0.0.0.0:5000 --chdir=/app -k uvicorn.workers.UvicornWorker
-/usr/local/bin/gunicorn wsgi --bind 0.0.0.0:5000 --chdir=/app --access-logfile - --error-logfile -
+{%- if cookiecutter.add_asgi.lower() == "y" %}
+gunicorn asgi --bind 0.0.0.0:8000 --chdir=/app -k uvicorn.workers.UvicornWorker
+{%- else %}
+gunicorn wsgi --bind 0.0.0.0:8000 --chdir=/app --access-logfile - --error-logfile -
+{%- endif %}

{{cookiecutter.github_repository}}/compose/local/start

@@ -6,5 +6,9 @@ set -o nounset
 
 
 python manage.py migrate
-#! uvicorn config.asgi:application --host 0.0.0.0 --reload
+
+{%- if cookiecutter.add_asgi.lower() == "y" %}
+uvicorn config.asgi:application --host 0.0.0.0 --reload
+{%- else %}
 python manage.py runserver_plus 0.0.0.0:8000
+{%- endif %}

{{cookiecutter.github_repository}}/docs/backend/server_config.md

@@ -4,13 +4,23 @@
 
 Our overall stack looks like this:
 
+{%- if cookiecutter.add_asgi.lower() == 'y' %}
+```
+the web client <-> the web server (nginx) <-> the socket <-> ASGI <-> Django
+```
+{%- else %}
 ```
 the web client <-> the web server (nginx) <-> the socket <-> uWSGI <-> Django
 ```
+{%- endif %}
 
 A web server faces the outside world. It can serve files (HTML, images, CSS, etc) directly from the file system. However, it can’t talk directly to Django applications; it needs something that will run the application, feed it requests from web clients (such as browsers) and return responses.
 
+{%- if cookiecutter.add_asgi.lower() == 'y' %}
+ASGI (ASGI stands for Asynchronous Server Gateway interface) which runs through Gunicorn running the actual Django instance. ASGI is an interface and sit in between the web server (NGINX) and the Django application. It creates a Unix socket, and serves responses to the web server via the asgi protocol.
+{%- else %}
 uWSGI is a [WSGI](https://en.wikipedia.org/wiki/Web_Server_Gateway_Interface) implementation, it creates a Unix socket, and serves responses to the web server via the uwsgi protocol.
+{%- endif %}
 
 ## Third Party Services
 

{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.443.conf.j2

@@ -51,11 +51,23 @@ server {
 
     # Setup named location for Django requests and handle proxy details
     location / {
+        {%- if cookiecutter.add_asgi.lower() == 'y' %}
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_pass http://uvicorn;
+
+        {%- else %}
         uwsgi_pass unix:///tmp/uwsgi-{{ project_namespace }}.sock;
         include /etc/nginx/uwsgi_params;
 
         # set correct scheme
         uwsgi_param UWSGI_SCHEME $http_x_forwarded_proto;
+        {%- endif %}
     }
 {% endraw %}
     {%- if cookiecutter.enable_whitenoise.lower() == 'n' %}
@@ -68,3 +80,14 @@ server {
     }{% endraw %}
     {%- endif %}
 }
+
+{%- if cookiecutter.add_asgi.lower() == 'y' %}
+upstream uvicorn {
+    {% raw %}server unix://{{ asgi_socket }};{% endraw %}
+}
+
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
+{%- endif %}

{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2

@@ -12,8 +12,13 @@ server {
     {% endif %}
 
     {% if vm and (nginx_cert.stat.exists == false or nginx_key.stat.exists == false) %}
-    location / {
-        uwsgi_pass unix:///tmp/uwsgi-{{ project_namespace }}.sock;
+    location / {{% endraw %}
+        {%- if cookiecutter.add_asgi.lower() == 'y' %}
+        {%raw%}proxy_pass unix://{{ asgi_socket }};{% endraw %}
+        {%- else %}
+        {%raw%}uwsgi_pass unix:///tmp/uwsgi-{{ project_namespace }}.sock;{% endraw %}
+        {%- endif %}
+        {% raw %}
         include /etc/nginx/uwsgi_params;
 
         # set correct scheme

{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml

@@ -4,7 +4,20 @@ pg_db: "{{ project_namespace }}"
 pg_user: dev
 pg_password: password
 django_requirements_file: requirements.txt
+{% endraw %}
 
+{%- if cookiecutter.add_asgi.lower() == 'y' %}
+# asgi related variables
+asgi_user: www-data
+asgi_group: www-data
+asgi_workers: 2
+{% raw %}
+asgi_socket: /tmp/django-{{ domain_name }}-asgi.sock
+{% endraw %}
+asgi_user: www-data
+asgi_group: www-data
+asgi_workers: 2
+{% else %}
 # uwsgi related variables
 uwsgi_user: www-data
 uwsgi_group: www-data
@@ -19,9 +32,11 @@ uwsgi_keepalive: 2
 uwsgi_loglevel: info
 uwsgi_conf_path: /etc/uwsgi-emperor/vassals
 uwsgi_emperor_pid_file: /run/uwsgi-emperor.pid
+{% raw %}
 uwsgi_socket: "/tmp/uwsgi-{{ project_namespace }}.sock"
 uwsgi_pid_file: "/tmp/uwsgi-{{ project_namespace }}.pid"
 
 uwsgi_log_dir: /var/log/uwsgi
 uwsgi_log_file: "{{ uwsgi_log_dir }}/{{ project_namespace }}.log"
 {% endraw %}
+{% endif %}

{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/asgi-setup.yml

@@ -0,0 +1,29 @@
+{% raw %}---
+- name: apt_get install asgi packages
+  apt: pkg={{ item }} state=present
+  with_items:
+    - uuid-dev
+    - libcap-dev
+    - libpcre3-dev
+  tags: ["configure"]
+
+- name: make sure project directory is owned by asgi group
+  file: path={{ project_path }} state=directory owner={{user}} group={{asgi_group}} recurse=yes
+  tags: ["configure"]
+
+- name: copy django-asgi logrotate
+  template: src=django.logrotate.j2
+    dest=/etc/logrotate.d/asgi-{{ deploy_environment}}-{{project_name}}-django
+    mode=644
+  tags: ["configure"]
+
+- name: make sure log directory exists
+  file: path={{ project_log_dir }} state=directory owner={{asgi_user}} group={{asgi_group}} mode=751 recurse=yes
+  tags: ["configure"]
+
+- name: copy Django asgi service to systemd
+  template: src=django.asgi.ini.j2
+    dest=/etc/systemd/system/asgi-{{project_namespace}}.service
+    mode=644
+  tags: ["deploy"]
+{% endraw %}

{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/main.yml

@@ -48,18 +48,30 @@
   become: false
   tags: ['deploy']
 
-- import_tasks: uwsgi-setup.yml
-
 - name: Run compilemessages for static translations
   django_manage: command=compilemessages app_path={{ project_path }} virtualenv={{ venv_path }}
   become: false
   tags: ['deploy']
 
+{% endraw %}
+{%- if cookiecutter.add_asgi.lower() == 'y' %}
+- import_tasks: asgi-setup.yml
+
+- name: Reload asgi processes
+{% raw %}
+  systemd: state=restarted name=asgi-{{ project_namespace }}
+{% endraw %}
+{%- else %}
+- import_tasks: uwsgi-setup.yml
+
+{% raw %}
 - name: Reload uwsgi processes
   command: uwsgi --reload {{ uwsgi_pid_file }}
   become: true
   when: not uwsgiconf.changed
-  tags: ['deploy']{% endraw %}
+{% endraw %}
+{%- endif %}
+  tags: ['deploy']
   {%- if cookiecutter.add_celery.lower() == 'y' %}
   notify: reload celery  # reload celery everytime uwsgi conf changes
   {%- endif %}

{{cookiecutter.github_repository}}/provisioner/roles/project_data/templates/django.asgi.ini.j2

@@ -0,0 +1,18 @@
+{% raw %}[Unit]
+Description={{ project_namespace }} gunicorn daemon
+After=network.target
+
+[Service]
+Environment=LC_ALL=en_US.utf-8
+Environment=LANG=en_US.utf-8
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=gunicorn
+User={{ asgi_user }}
+Group={{ asgi_group }}
+WorkingDirectory={{ project_path }}
+ExecStart={{ venv_path }}/bin/gunicorn -w {{ asgi_workers }} --bind unix://{{ asgi_socket }} --access-logfile {{project_log_dir}}/asgi.log --capture-output --error-logfile {{project_log_dir}}/asgi-errors.log -k uvicorn.workers.UvicornWorker asgi:application
+
+[Install]
+WantedBy=multi-user.target
+{% endraw %}

{{cookiecutter.github_repository}}/pyproject.toml

@@ -6,15 +6,15 @@ authors = ["{{cookiecutter.default_from_email}}"]
 
 [tool.poetry.dependencies]
 python = "~3.9"
-Django = "~3.2.15"
+Django = "~4.1"
 django-environ = "^0.9"
 django-sites = "^0.11"
 django-filter = "^21.1"
 argon2-cffi = "^21.3"
 python-dotenv = "^0.21"
 django-cors-headers = "^3.13"
 {% if cookiecutter.enable_whitenoise.lower() == 'y' -%}
-whitenoise = "^6.2"
+whitenoise = "^6.4.0"
 {%- endif %}
 
 # Extensions
@@ -32,7 +32,7 @@ django-versatileimagefield = "^2.2"
 
 # REST APIs
 # -------------------------------------
-djangorestframework = "3.13.1"
+djangorestframework = "3.14"
 drf-yasg = "^1.21"
 
 
@@ -78,6 +78,9 @@ django-mail-templated = "^2.6"
 # Static Files and Media Storage
 # -------------------------------------
 gunicorn = "~20.1.0"
+{%- if cookiecutter.add_asgi.lower() == "y" %}
+uvicorn = "^0.21.0"
+{%- endif %}
 django-storages = "^1.13"
 boto3 = "~1.26.47"
 

{{cookiecutter.github_repository}}/{{cookiecutter.main_module}}/graphql/api.py

@@ -2,7 +2,7 @@
 import graphene
 from graphene_django.debug import DjangoDebug
 
-from .users.schema import UserQueries, UserMutations
+from .users.schema import UserMutations, UserQueries
 
 
 class Query(UserQueries):

{{cookiecutter.github_repository}}/{{cookiecutter.main_module}}/graphql/decorators.py

@@ -4,7 +4,7 @@
 try:
     from graphql.execution.execute import GraphQLResolveInfo
 except ImportError:
-    from graphql.execution.base import ResolveInfo as GraphQLResolveInfo
+    from graphql.execution.base import ResolveInfo as GraphQLResolveInfo  # type: ignore
 
 
 def context(f):

{{cookiecutter.github_repository}}/{{cookiecutter.main_module}}/graphql/middleware.py

@@ -1,4 +1,3 @@
-from django.contrib.auth import authenticate
 from {{cookiecutter.main_module}}.users.auth.tokens import get_user_for_token
 from {{cookiecutter.main_module}}.users.auth.utils import get_http_authorization
 

{{cookiecutter.github_repository}}/{{cookiecutter.main_module}}/graphql/users/mutations.py

@@ -4,9 +4,10 @@
 from graphene import relay
 from graphql import GraphQLError
 
-from .types import CurrentUser, AuthenticatedUser
 from {{cookiecutter.main_module}}.users import services as user_services
-from {{cookiecutter.main_module}}.users.auth import tokens, services as auth_services
+from {{cookiecutter.main_module}}.users.auth import tokens
+from {{cookiecutter.main_module}}.users.auth import services as auth_services
+from .types import AuthenticatedUser, CurrentUser
 
 
 class SignUp(relay.ClientIDMutation):
@@ -44,11 +45,12 @@ def validate_email(email):
     user = graphene.Field(AuthenticatedUser)
 
     @classmethod
-    def mutate_and_get_payload(cls, root, info, **data):
-        cls.validate_email(data["email"])
-        user = user_services.get_and_authenticate_user(**data)
+    def mutate_and_get_payload(cls, root, info, email, password):
+        cls.validate_email(email)
+        user = user_services.get_and_authenticate_user(email, password)
         return Login(user=user)
 
+
 class PasswordChange(relay.ClientIDMutation):
     class Input:
         current_password = graphene.String(required=True)
@@ -57,10 +59,8 @@ class Input:
     user = graphene.Field(AuthenticatedUser)
 
     @classmethod
-    def mutate_and_get_payload(cls, root, info, **data):
+    def mutate_and_get_payload(cls, root, info, current_password, new_password):
         user = info.context.user
-        current_password = data["current_password"]
-        new_password = data["new_password"]
 
         if not user.check_password(current_password):
             raise GraphQLError("invalid_password")
@@ -91,8 +91,7 @@ def clean_user(cls, email):
         return user
 
     @classmethod
-    def mutate_and_get_payload(cls, root, info, **data):
-        email = data["email"]
+    def mutate_and_get_payload(cls, root, info, email):
         user = cls.clean_user(email)
 
         auth_services.send_password_reset_mail(user)
@@ -109,9 +108,7 @@ class Input:
     message = graphene.String()
 
     @classmethod
-    def mutate_and_get_payload(cls, root, info, **data):
-        new_password = data["new_password"]
-        token = data["token"]
+    def mutate_and_get_payload(cls, root, info, token, new_password):
 
         user = tokens.get_user_for_password_reset_token(token)
         password_validation.validate_password(new_password, user)