diff --git a/arches/__init__.py b/arches/__init__.py index 7fd8f2fe68a..d47d2b40aab 100644 --- a/arches/__init__.py +++ b/arches/__init__.py @@ -1,5 +1,15 @@ +from __future__ import absolute_import from arches.setup import get_version -VERSION = (4, 4, 1, 'final', 0) +try: + from .celery import app as celery_app +except ModuleNotFoundError as e: + print(e) + +VERSION = (5, 0, 0, 'final', 0) __version__ = get_version(VERSION) + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +__all__ = ('celery_app',) diff --git a/arches/app/tasks.py b/arches/app/tasks.py new file mode 100644 index 00000000000..04d89f0fe2a --- /dev/null +++ b/arches/app/tasks.py @@ -0,0 +1,9 @@ +# Create your tasks here +from __future__ import absolute_import, unicode_literals +from celery import shared_task +from django.core import management + +@shared_task +def sync(surveyid, userid): + management.call_command('mobile', operation='sync_survey', id=surveyid, user=userid) + return 'sync complete' diff --git a/arches/app/utils/task_management.py b/arches/app/utils/task_management.py new file mode 100644 index 00000000000..fc0d463a5ad --- /dev/null +++ b/arches/app/utils/task_management.py @@ -0,0 +1,24 @@ +import logging +from kombu import Connection +from django.utils.translation import ugettext as _ +from arches.app.models.system_settings import settings +from arches.celery import app + +logger = logging.getLogger(__name__) + + +def check_if_celery_available(): + try: + conn = Connection(settings.CELERY_BROKER_URL) + conn.ensure_connection(max_retries=2) + except Exception as e: + logger.warning(_("Unable to connect to a celery broker")) + return False + inspect = app.control.inspect() + result = inspect.ping() + if result is None: + logger.warning(_("A celery broker is running, but a celery worker is not available")) + result = False # ping returns True or None, assigning False here so we return only a boolean value + else: + result = True + return result diff --git a/arches/app/views/api.py b/arches/app/views/api.py index 01519a31e05..6a496d6b48a 100644 --- a/arches/app/views/api.py +++ b/arches/app/views/api.py @@ -4,6 +4,8 @@ import re import sys import uuid +import arches.app.tasks as tasks +import arches.app.utils.task_management as task_management from io import StringIO from django.shortcuts import render from django.views.generic import View @@ -39,6 +41,7 @@ from pyld.jsonld import compact, frame, from_rdf from rdflib import RDF from rdflib.namespace import SKOS, DCTERMS +from arches.celery import app logger = logging.getLogger(__name__) @@ -123,7 +126,11 @@ def get(self, request, surveyid=None): if can_sync: try: logger.info("Starting sync for user {0}".format(request.user.username)) - management.call_command('mobile', operation='sync_survey', id=surveyid, user=request.user.id) + celery_worker_running = task_management.check_if_celery_available() + if celery_worker_running is True: + tasks.sync.delay(surveyid=surveyid, userid=request.user.id) + else: + management.call_command('mobile', operation='sync_survey', id=surveyid, user=request.user.id) logger.info("Sync complete for user {0}".format(request.user.username)) except Exception: logger.exception(_('Sync Failed')) diff --git a/arches/celery.py b/arches/celery.py new file mode 100644 index 00000000000..bb5516f5913 --- /dev/null +++ b/arches/celery.py @@ -0,0 +1,22 @@ +from __future__ import absolute_import, unicode_literals +import os +import celery +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') + +app = Celery('arches') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) diff --git a/arches/install/arches-templates/project_name/celery.py-tpl b/arches/install/arches-templates/project_name/celery.py-tpl new file mode 100644 index 00000000000..c96bac12fdc --- /dev/null +++ b/arches/install/arches-templates/project_name/celery.py-tpl @@ -0,0 +1,21 @@ +from __future__ import absolute_import, unicode_literals +import os +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', '{{ project_name }}.settings') + +app = Celery('{{ project_name }}') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) diff --git a/arches/install/arches-templates/project_name/settings.py-tpl b/arches/install/arches-templates/project_name/settings.py-tpl index 632f6fbbd09..00983e87407 100644 --- a/arches/install/arches-templates/project_name/settings.py-tpl +++ b/arches/install/arches-templates/project_name/settings.py-tpl @@ -73,6 +73,7 @@ INSTALLED_APPS = ( 'revproxy', 'corsheaders', 'oauth2_provider', + 'django_celery_results', '{{ project_name }}', ) @@ -147,12 +148,17 @@ APP_TITLE = 'Arches | Heritage Data Management' COPYRIGHT_TEXT = 'All Rights Reserved.' COPYRIGHT_YEAR = '2019' +CELERY_BROKER_URL = 'amqp://guest:guest@localhost' +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_RESULT_BACKEND = 'django-db' # Use 'django-cache' if you want to use your cache as your backend +CELERY_TASK_SERIALIZER = 'json' + try: - from package_settings import * + from .package_settings import * except ImportError: pass try: - from settings_local import * + from .settings_local import * except ImportError: pass diff --git a/arches/install/arches-templates/project_name/settings_local.py-tpl b/arches/install/arches-templates/project_name/settings_local.py-tpl index 9652c2c5002..da8f3eaede9 100644 --- a/arches/install/arches-templates/project_name/settings_local.py-tpl +++ b/arches/install/arches-templates/project_name/settings_local.py-tpl @@ -1,315 +1,4 @@ -import os - -#ADMINS = ( -#) - - -#ADMIN_MEDIA_PREFIX = '/media/admin/' - -#ANALYSIS_COORDINATE_SYSTEM_SRID = 3857 - -#ANONYMOUS_USER_NAME = None - -#APP_NAME = 'Arches' - -#APP_TITLE = 'Arches | Heritage Data Management' - - -#AUTHENTICATION_BACKENDS = ( -# oauth2_provider.backends.OAuth2Backend, -# django.contrib.auth.backends.ModelBackend, -# guardian.backends.ObjectPermissionBackend, -# arches.app.utils.permission_backend.PermissionBackend, -#) - - - -#AUTH_PASSWORD_VALIDATORS = [ -# { -# "NAME": "arches.app.utils.password_validation.NumericPasswordValidator" -# }, -# { -# "NAME": "arches.app.utils.password_validation.SpecialCharacterValidator", -# "OPTIONS": { -# "special_characters": [ -# "!", -# "@", -# "#", -# ")", -# "(", -# "*", -# "&", -# "^", -# "%", -# "$" -# ] -# } -# }, -# { -# "NAME": "arches.app.utils.password_validation.HasNumericCharacterValidator" -# }, -# { -# "NAME": "arches.app.utils.password_validation.HasUpperAndLowerCaseValidator" -# }, -# { -# "NAME": "arches.app.utils.password_validation.MinLengthValidator", -# "OPTIONS": { -# "min_length": 9 -# } -# } -#] - - - -#CACHES = { -# "default": { -# "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", -# "LOCATION": "127.0.0.1:11211" -# } -#} - - - -#CACHE_BY_USER = { -# "anonymous": 86400 -#} - - -#COPYRIGHT_TEXT = 'All Rights Reserved.' - -#COPYRIGHT_YEAR = '2016' - - -#DATABASES = { -# "default": { -# "ATOMIC_REQUESTS": False, -# "AUTOCOMMIT": True, -# "CONN_MAX_AGE": 0, -# "ENGINE": "django.contrib.gis.db.backends.postgis", -# "HOST": "localhost", -# "NAME": "arches", -# "OPTIONS": {}, -# "PASSWORD": "postgis", -# "PORT": "5432", -# "POSTGIS_TEMPLATE": "template_postgis", -# "TEST": { -# "CHARSET": None, -# "COLLATION": None, -# "MIRROR": None, -# "NAME": None -# }, -# "TIME_ZONE": None, -# "USER": "postgres" -# } -#} - - - -#DATATYPE_LOCATIONS = [ -# "arches.app.datatypes" -#] - - -#DATE_IMPORT_EXPORT_FORMAT = '%Y-%m-%d' - -#DEBUG = True - - -#ELASTICSEARCH_CONNECTION_OPTIONS = { -# "timeout": 30 -#} - - - -#ELASTICSEARCH_HOSTS = [ -# { -# "host": "localhost", -# "port": 9200 -# } -#] - - -#ELASTICSEARCH_HTTP_PORT = 9200 - -#EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' - -#EMAIL_HOST = 'localhost' - -#EMAIL_HOST_PASSWORD = '' - -#EMAIL_HOST_USER = '' - -#EMAIL_PORT = 25 - -#EMAIL_USE_TLS = False - - -#INSTALLED_APPS = ( -# django.contrib.admin, -# django.contrib.auth, -# django.contrib.contenttypes, -# django.contrib.sessions, -# django.contrib.messages, -# django.contrib.staticfiles, -# django.contrib.gis, -# arches, -# arches.app.models, -# arches.management, -# guardian, -# captcha, -# revproxy, -# corsheaders, -# oauth2_provider, -#) - - - -#INTERNAL_IPS = ( -# 127.0.0.1, -#) - - -#LANGUAGE_CODE = 'en-US' - - -#LOCALE_PATHS = [ -# /arches/locale, -#] - - - -#LOGGING = { -# "disable_existing_loggers": False, -# "handlers": { -# "file": { -# "class": "logging.FileHandler", -# "filename": "/arches/arches.log", -# "level": "DEBUG" -# } -# }, -# "loggers": { -# "arches": { -# "handlers": [ -# "file" -# ], -# "level": "DEBUG", -# "propagate": True -# } -# }, -# "version": 1 -#} - - -#LOGIN_URL = 'auth' - - -#MANAGERS = ( -#) - - -#MEDIA_ROOT = '/arches' - -#MEDIA_URL = '/files/' - - -#MIDDLEWARE_CLASSES = [ -# "django.middleware.common.CommonMiddleware", -# "django.middleware.csrf.CsrfViewMiddleware" -#] - - -#MODE = 'PROD' - -#ONTOLOGY_BASE = 'cidoc_crm_v6.2.xml' - -#ONTOLOGY_BASE_ID = 'e6e8db47-2ccf-11e6-927e-b8f6b115d7dd' - -#ONTOLOGY_BASE_NAME = 'CIDOC CRM v6.2' - -#ONTOLOGY_BASE_VERSION = '6.2' - - -#ONTOLOGY_EXT = [ -# "CRMsci_v1.2.3.rdfs.xml", -# "CRMarchaeo_v1.4.rdfs.xml", -# "CRMgeo_v1.2.rdfs.xml", -# "CRMdig_v3.2.1.rdfs.xml", -# "CRMinf_v0.7.rdfs.xml", -# "arches_crm_enhancements.xml" -#] - - -#ONTOLOGY_PATH = '/arches/db/ontologies/cidoc_crm' - - -#POSTGIS_VERSION = ( -# 2, -# 0, -# 0, -#) - - -#RESOURCE_IMPORT_LOG = 'arches/logs/resource_import.log' - -#ROOT_DIR = '/arches' - -#ROOT_URLCONF = 'arches.urls' - -#SEARCH_BACKEND = 'arches.app.search.search.SearchEngine' - - -#STATICFILES_DIRS = ( -# /arches/app/media, -#) - - - -#STATICFILES_FINDERS = ( -# django.contrib.staticfiles.finders.FileSystemFinder, -# django.contrib.staticfiles.finders.AppDirectoriesFinder, -#) - - -#STATIC_ROOT = '' - -#STATIC_URL = '/media/' - -#SYSTEM_SETTINGS_LOCAL_PATH = '/arches/db/system_settings/Arches_System_Settings_Local.json' - - -#TEMPLATES = [ -# { -# "APP_DIRS": True, -# "BACKEND": "django.template.backends.django.DjangoTemplates", -# "DIRS": [ -# "/arches/app/templates" -# ], -# "OPTIONS": { -# "context_processors": [ -# "django.contrib.auth.context_processors.auth", -# "django.template.context_processors.debug", -# "django.template.context_processors.i18n", -# "django.template.context_processors.media", -# "django.template.context_processors.static", -# "django.template.context_processors.tz", -# "django.template.context_processors.request", -# "django.contrib.messages.context_processors.messages", -# "arches.app.utils.context_processors.livereload", -# "arches.app.utils.context_processors.map_info", -# "arches.app.utils.context_processors.app_settings" -# ], -# "debug": True -# } -# } -#] - - -#TIME_ZONE = 'America/Chicago' - -#USE_I18N = True - -#USE_L10N = True - -#USE_TZ = False - -#WSGI_APPLICATION = 'arches.wsgi.application' +try: + from .{{ project_name }}.settings import * +except ImportError: + pass diff --git a/arches/install/requirements.txt b/arches/install/requirements.txt index 72a17f1a672..4e5535d0ee6 100644 --- a/arches/install/requirements.txt +++ b/arches/install/requirements.txt @@ -4,6 +4,8 @@ elasticsearch>=7.0.0,<8.0.0 rdflib==4.2.2 django-guardian==2.1.0 python-memcached==1.59 +celery==4.3.0 +django-celery-results==1.1.2 mapbox-vector-tile==1.2.0 SPARQLWrapper==1.8.4 django-recaptcha==2.0.5 diff --git a/arches/management/commands/mobile.py b/arches/management/commands/mobile.py index d78cc29b450..85bd054c7bf 100644 --- a/arches/management/commands/mobile.py +++ b/arches/management/commands/mobile.py @@ -49,7 +49,7 @@ def add_arguments(self, parser): '\'rebuild_surveys\' rebuilds all surveys that belong to the current arches install') parser.add_argument('-id', '--id', dest='id', default=None, help='UUID of Survey') - parser.add_argument('-user', '--user', dest='user', default=None, + parser.add_argument('-u', '--user', dest='user', default=None, help='UUID of Survey') def handle(self, *args, **options): diff --git a/arches/settings.py b/arches/settings.py index 24074cb0f7e..6d9b9eb31d8 100644 --- a/arches/settings.py +++ b/arches/settings.py @@ -303,6 +303,7 @@ 'revproxy', 'corsheaders', 'oauth2_provider', + 'django_celery_results', #'debug_toolbar' ) @@ -560,6 +561,11 @@ ALLOWED_POPUP_HOSTS = [] TILESERVER_URL = None + +CELERY_BROKER_URL = 'amqp://guest:guest@localhost' +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_RESULT_BACKEND = 'django-db' # Use 'django-cache' if you want to use your cache as your backend +CELERY_TASK_SERIALIZER = 'json' ########################################## ### END RUN TIME CONFIGURABLE SETTINGS ### ########################################## diff --git a/arches/urls.py b/arches/urls.py index 8b1c246fb9a..aa21d5b4e90 100644 --- a/arches/urls.py +++ b/arches/urls.py @@ -118,7 +118,6 @@ url(r'^resource/descriptors/(?P%s|())$' % uuid_regex, ResourceDescriptors.as_view(), name="resource_descriptors"), url(r'^resource/(?P%s)/tiles$' % uuid_regex, ResourceTiles.as_view(), name='resource_tiles'), url(r'^report/(?P%s)$' % uuid_regex, ResourceReportView.as_view(), name='resource_report'), - url(r'^report/(?P%s)$' % uuid_regex, ResourceReportView.as_view(), name='resource_report'), url(r'^card/(?P%s|())$' % uuid_regex, CardView.as_view(action='update_card'), name='card'), url(r'^reorder_cards/', CardView.as_view(action='reorder_cards'), name='reorder_cards'), url(r'^node/(?P%s)$' % uuid_regex, GraphDataView.as_view(action='update_node'), name='node'), diff --git a/tests/models/tile_model_tests.py b/tests/models/tile_model_tests.py index ddf0b7bbca1..0b814843580 100644 --- a/tests/models/tile_model_tests.py +++ b/tests/models/tile_model_tests.py @@ -48,17 +48,17 @@ def setUpClass(cls): INSERT INTO public.resource_instances(resourceinstanceid, legacyid, graphid, createdtime) VALUES ('40000000-0000-0000-0000-000000000000', '40000000-0000-0000-0000-000000000000', '2f7f8e40-adbc-11e6-ac7f-14109fd34195', '1/1/2000'); - INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality) - VALUES ('99999999-0000-0000-0000-000000000001', '', 'n'); + INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality, exportable) + VALUES ('99999999-0000-0000-0000-000000000001', '', 'n', false); - INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality) - VALUES ('32999999-0000-0000-0000-000000000000', '', 'n'); + INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality, exportable) + VALUES ('32999999-0000-0000-0000-000000000000', '', 'n', false); - INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality) - VALUES ('19999999-0000-0000-0000-000000000000', '', 'n'); + INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality, exportable) + VALUES ('19999999-0000-0000-0000-000000000000', '', 'n', false); - INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality) - VALUES ('21111111-0000-0000-0000-000000000000', '', 'n'); + INSERT INTO node_groups(nodegroupid, legacygroupid, cardinality, exportable) + VALUES ('21111111-0000-0000-0000-000000000000', '', 'n', false); """ cursor = connection.cursor()