Merge pull request #643 from vintasoftware/recommended-celery-settings

Apply recommended Celery Django settings for reliability

README.md

@@ -304,9 +304,9 @@ Some settings defaults were decided based on Vinta's experiences. Here's the rat
 - **Important:** When you are queueing a new Celery task directly from a Django view, particularly with little or no delay/ETA, it is essential to use `transaction.on_commit(lambda: my_task.delay())`. This ensures that the task is only queued after the associated database transaction has been successfully committed.
   - If `transaction.on_commit` is not utilized, or if a significant delay is not set, you risk encountering race conditions. In such scenarios, the Celery task might execute before the completion of the request's transaction. This can lead to inconsistencies and unexpected behavior, as the task might operate on a database state that does not yet reflect the changes made in the transaction. Read more about this problem on [this article](https://www.vinta.com.br/blog/database-concurrency-in-django-the-right-way).
 
-### `CELERY_ACKS_LATE = True`
+### `CELERY_TASK_ACKS_LATE = True`
 
-- We believe Celery tasks should be idempotent. So for us it's safe to set `CELERY_ACKS_LATE = True` to ensure tasks will be re-queued after a worker failure. Check Celery docs on ["Should I use retry or acks_late?"](https://docs.celeryq.dev/en/stable/faq.html#faq-acks-late-vs-retry) for more info.
+- We believe Celery tasks should be idempotent. So for us it's safe to set `CELERY_TASK_ACKS_LATE = True` to ensure tasks will be re-queued after a worker failure. Check Celery docs on ["Should I use retry or acks_late?"](https://docs.celeryq.dev/en/stable/faq.html#faq-acks-late-vs-retry) for more info.
 
 ## Contributing
 

backend/project_name/settings/base.py

@@ -132,11 +132,36 @@ def base_dir_join(*args):
 }
 
 # Celery
+# Recommended settings for reliability: https://gist.github.com/fjsj/da41321ac96cf28a96235cb20e7236f6
 CELERY_ACCEPT_CONTENT = ["json"]
 CELERY_TASK_SERIALIZER = "json"
 CELERY_RESULT_SERIALIZER = "json"
-CELERY_ACKS_LATE = True
+CELERY_TASK_ACKS_LATE = True
 CELERY_TIMEZONE = TIME_ZONE
+CELERY_BROKER_TRANSPORT_OPTIONS = {"confirm_publish": True, "confirm_timeout": 5.0}
+CELERY_BROKER_POOL_LIMIT = config("CELERY_BROKER_POOL_LIMIT", cast=int, default=1)
+CELERY_BROKER_CONNECTION_TIMEOUT = config(
+    "CELERY_BROKER_CONNECTION_TIMEOUT", cast=float, default=30.0
+)
+CELERY_REDIS_MAX_CONNECTIONS = config(
+    "CELERY_REDIS_MAX_CONNECTIONS", cast=lambda v: int(v) if v else None, default=None
+)
+CELERY_TASK_ACKS_ON_FAILURE_OR_TIMEOUT = config(
+    "CELERY_TASK_ACKS_ON_FAILURE_OR_TIMEOUT", cast=bool, default=True
+)
+CELERY_TASK_REJECT_ON_WORKER_LOST = config(
+    "CELERY_TASK_REJECT_ON_WORKER_LOST", cast=bool, default=False
+)
+CELERY_WORKER_PREFETCH_MULTIPLIER = config("CELERY_WORKER_PREFETCH_MULTIPLIER", cast=int, default=1)
+CELERY_WORKER_CONCURRENCY = config(
+    "CELERY_WORKER_CONCURRENCY", cast=lambda v: int(v) if v else None, default=None
+)
+CELERY_WORKER_MAX_TASKS_PER_CHILD = config(
+    "CELERY_WORKER_MAX_TASKS_PER_CHILD", cast=int, default=1000
+)
+CELERY_WORKER_SEND_TASK_EVENTS = config("CELERY_WORKER_SEND_TASK_EVENTS", cast=bool, default=True)
+CELERY_EVENT_QUEUE_EXPIRES = config("CELERY_EVENT_QUEUE_EXPIRES", cast=float, default=60.0)
+CELERY_EVENT_QUEUE_TTL = config("CELERY_EVENT_QUEUE_TTL", cast=float, default=5.0)
 
 # Sentry
 SENTRY_DSN = config("SENTRY_DSN", default="")

backend/project_name/settings/production.py

@@ -44,7 +44,8 @@
 WEBPACK_LOADER["DEFAULT"]["CACHE"] = True
 
 # Celery
-CELERY_BROKER_URL = config("REDIS_URL")
+# Recommended settings for reliability: https://gist.github.com/fjsj/da41321ac96cf28a96235cb20e7236f6
+CELERY_BROKER_URL = config("RABBITMQ_URL", default="") or config("REDIS_URL")
 CELERY_RESULT_BACKEND = config("REDIS_URL")
 CELERY_SEND_TASK_ERROR_EMAILS = True
 

render.yaml

@@ -9,7 +9,7 @@ services:
     name: {{project_name}}-redis
     plan: free
     ipAllowList: [] # only allow internal connections
-    
+
   - type: web
     name: {{project_name}}
     plan: free
@@ -32,8 +32,8 @@ services:
           property: connectionString
       - fromGroup: python-services
       - fromGroup: integrations-credentials
-      
-  # As there aren't free plans for Workers in Render, the configuration for 
+
+  # As there aren't free plans for Workers in Render, the configuration for
   # celery workers/beat will be commented by default
   #
   # - type: worker
@@ -44,7 +44,29 @@ services:
   #   startCommand: "poetry run celery --workdir backend --app={{project_name}} worker --loglevel=info --max-memory-per-child=$WORKER_MAX_MEMORY  --concurrency=$WORKER_CONCURRENCY"
   #   envVars:
   #     - key: REMAP_SIGTERM
-  #       value: SIGQUIT      
+  #       value: SIGQUIT
+  #     - key: CELERY_BROKER_POOL_LIMIT
+  #       value: 1
+  #     - key: CELERY_BROKER_CONNECTION_TIMEOUT
+  #       value: 30.0
+  #     - key: CELERY_REDIS_MAX_CONNECTIONS
+  #       value: null
+  #     - key: CELERY_TASK_ACKS_ON_FAILURE_OR_TIMEOUT
+  #       value: true
+  #     - key: CELERY_TASK_REJECT_ON_WORKER_LOST
+  #       value: false
+  #     - key: CELERY_WORKER_PREFETCH_MULTIPLIER
+  #       value: 1
+  #     - key: CELERY_WORKER_CONCURRENCY
+  #       value: null
+  #     - key: CELERY_WORKER_MAX_TASKS_PER_CHILD
+  #       value: 1000
+  #     - key: CELERY_WORKER_SEND_TASK_EVENTS
+  #       value: true
+  #     - key: CELERY_EVENT_QUEUE_EXPIRES
+  #       value: 60.0
+  #     - key: CELERY_EVENT_QUEUE_TTL
+  #       value: 5.0
   #     - key: WORKER_MAX_MEMORY
   #       sync: false
   #     - key: WORKER_CONCURRENCY