add celery task for update (#482)

.env.example

@@ -20,3 +20,5 @@ DJANGO_SUPERUSER_PASSWORD='admin'
 CACHE_LOCATION='memcached:11211'
 CACHE_USERNAME=
 CACHE_PASSWORD=
+BROKER_URL='amqp://guest:guest@172.17.0.1:5672//'
+RESULT_BACKEND_URL='rpc://'

Procfile

@@ -1,2 +1,4 @@
 release: python manage.py migrate
 web: gunicorn config.wsgi --log-file -
+worker: celery -A config worker -l info --concurrency=4
+beat: celery -A config beat -l info

config/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

config/celery.py

@@ -0,0 +1,19 @@
+import os
+
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
+
+app = Celery("config")
+
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Auto-discover tasks in all installed apps
+# This will look for a 'tasks.py' file in each app directory
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+    """A simple task for testing Celery setup"""
+    print(f"Request: {self.request!r}")

config/settings.py

@@ -4,10 +4,14 @@
 
 import dj_database_url
 import sentry_sdk
+from celery.schedules import crontab
 from sentry_sdk.integrations.django import DjangoIntegration
+from dotenv import load_dotenv
 
 from contributors.utils import misc
 
+load_dotenv()
+
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 # Logging setup
@@ -291,3 +295,19 @@
     }
 
 SITE_ID = 1
+
+# Just broker URL, no result backend needed
+CELERY_BROKER_URL = os.getenv('BROKER_URL', 'amqp://guest:guest@172.17.0.1:5672//')
+CELERY_RESULT_BACKEND = None
+CELERY_IGNORE_RESULT = True
+
+CELERY_BEAT_SCHEDULE = {
+    'sync-github-repositories': {
+        'task': 'contributors.tasks.sync_github_data',
+        'schedule': crontab(hour='*/6'),
+        'options': {
+            'expires': 3600,
+            'time_limit': 3600,
+        }
+    },
+}

contributors/tasks.py

@@ -0,0 +1,19 @@
+from celery import shared_task
+from celery.utils.log import get_task_logger
+from django.core.management import call_command
+
+from contributors.management.commands.fetchdata import ORGANIZATIONS
+
+logger = get_task_logger(__name__)
+
+
+@shared_task(bind=True, max_retries=3)
+def sync_github_data(self, owners=None, repos=None):
+    try:
+        orgs = [org.name for org in ORGANIZATIONS]
+        logger.info(orgs)
+        call_command("fetchdata", orgs or ["hexlet"], repo=repos or [])
+
+    except Exception as exc:
+        logger.error(f"Failed to sync GitHub data: {exc}")
+        self.retry(exc=exc, countdown=60 * (2**self.request.retries))

contributors/utils/misc.py

@@ -16,7 +16,7 @@
 def getenv(env_variable):
     """Return an environment variable or raise an exception."""
     try:
-        return os.environ[env_variable]
+        return os.getenv(env_variable)
     except KeyError:
         raise ImproperlyConfigured(
             f"The {env_variable} setting must not be empty.",
@@ -34,51 +34,51 @@ def update_or_create_record(cls, github_resp, additional_fields=None):
 
     """
     cls_fields = {
-        'Organization': lambda: {'name': github_resp['login']},
-        'Repository': lambda: {
-            'owner_id': (
-                github_resp['owner']['id']
-                if github_resp['owner']['type'] == 'User'
+        "Organization": lambda: {"name": github_resp["login"]},
+        "Repository": lambda: {
+            "owner_id": (
+                github_resp["owner"]["id"]
+                if github_resp["owner"]["type"] == "User"
                 else None
             ),
-            'organization_id': (
-                github_resp['owner']['id']
-                if github_resp['owner']['type'] == 'Organization'
+            "organization_id": (
+                github_resp["owner"]["id"]
+                if github_resp["owner"]["type"] == "Organization"
                 else None
             ),
-            'full_name': github_resp['full_name'],
+            "full_name": github_resp["full_name"],
         },
-        'Contributor': lambda: {
-            'login': github_resp['login'],
-            'avatar_url': github_resp['avatar_url'],
+        "Contributor": lambda: {
+            "login": github_resp["login"],
+            "avatar_url": github_resp["avatar_url"],
         },
     }
     defaults = {
-        'name': github_resp.get('name'),
-        'html_url': github_resp['html_url'],
+        "name": github_resp.get("name"),
+        "html_url": github_resp["html_url"],
     }
 
     defaults.update(cls_fields[cls.__name__]())
     defaults.update(additional_fields or {})
     return cls.objects.update_or_create(
-        id=github_resp['id'],
+        id=github_resp["id"],
         defaults=defaults,
     )
 
 
 def get_contributor_data(login, session=None):
     """Get contributor data from database or GitHub."""
-    Contributor = apps.get_model('contributors.Contributor')    # noqa: N806
+    Contributor = apps.get_model("contributors.Contributor")  # noqa: N806
     try:
         user = Contributor.objects.get(login=login)
     except Contributor.DoesNotExist:
         return github.get_owner_data(login, session)
     return {
-        'id': user.id,
-        'name': user.name,
-        'html_url': user.html_url,
-        'login': user.login,
-        'avatar_url': user.avatar_url,
+        "id": user.id,
+        "name": user.name,
+        "html_url": user.html_url,
+        "login": user.login,
+        "avatar_url": user.avatar_url,
     }
 
 
@@ -100,13 +100,15 @@ def group_contribs_by_months(months_with_contrib_sums):
     """
     sums_of_contribs_by_months = {}
     for contrib in months_with_contrib_sums:
-        month = sums_of_contribs_by_months.setdefault(contrib['month'], {})
-        month[contrib['type']] = contrib['count']
+        month = sums_of_contribs_by_months.setdefault(contrib["month"], {})
+        month[contrib["type"]] = contrib["count"]
     return sums_of_contribs_by_months
 
 
 def get_rotated_sums_for_contrib(
-    current_month: int, sums_of_contribs_by_months, contrib_type,
+    current_month: int,
+    sums_of_contribs_by_months,
+    contrib_type,
 ):
     """
     Return an array of 12 sums of contributions of the given type.
@@ -115,16 +117,19 @@ def get_rotated_sums_for_contrib(
     The collection is left-shifted by the numeric value of the current month.
     """
     months = range(1, NUM_OF_MONTHS_IN_A_YEAR + 1)
-    array = deque([
-        sums_of_contribs_by_months.get(month, {}).get(contrib_type, 0)
-        for month in months
-    ])
+    array = deque(
+        [
+            sums_of_contribs_by_months.get(month, {}).get(contrib_type, 0)
+            for month in months
+        ]
+    )
     array.rotate(-current_month)
     return list(array)
 
 
 def get_contrib_sums_distributed_over_months(
-    current_month: int, sums_of_contribs_by_months,
+    current_month: int,
+    sums_of_contribs_by_months,
 ):
     """
     Return shifted monthly sums for each contribution type.
@@ -137,10 +142,10 @@ def get_contrib_sums_distributed_over_months(
         sums_of_contribs_by_months,
     )
     return {
-        'commits': rotated_sums_by_months('cit'),
-        'pull_requests': rotated_sums_by_months('pr'),
-        'issues': rotated_sums_by_months('iss'),
-        'comments': rotated_sums_by_months('cnt'),
+        "commits": rotated_sums_by_months("cit"),
+        "pull_requests": rotated_sums_by_months("pr"),
+        "issues": rotated_sums_by_months("iss"),
+        "comments": rotated_sums_by_months("cnt"),
     }
 
 
@@ -159,21 +164,23 @@ def datetime_week_ago():
 def split_full_name(name):
     """Split a full name into parts."""
     if not name:
-        return ('', '')
+        return ("", "")
     name_parts = name.split()
     first_name = name_parts[0]
-    last_name = name_parts[-1] if len(name_parts) > 1 else ''
+    last_name = name_parts[-1] if len(name_parts) > 1 else ""
     return (first_name, last_name)
 
 
 def split_ordering(ordering):
     """Return a tuple of ordering direction and field name."""
-    if ordering.startswith('-'):
-        return ('-', ordering[1:])
-    return ('', ordering)
+    if ordering.startswith("-"):
+        return ("-", ordering[1:])
+    return ("", ordering)
 
 
-DIRECTION_TRANSLATIONS = MappingProxyType({
-    '': 'asc',
-    '-': 'desc',
-})
+DIRECTION_TRANSLATIONS = MappingProxyType(
+    {
+        "": "asc",
+        "-": "desc",
+    }
+)

pyproject.toml

@@ -6,6 +6,7 @@ readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
     "apscheduler>=3.10.4",
+    "celery>=5.4.0",
     "crispy-bootstrap5>=2024.10",
     "cryptography>=43.0.3",
     "dj-database-url>=2.3.0",