tweaked stats update script

victorverma3 · Jan 22, 2025 · 139b6bb · 139b6bb
1 parent c723308
commit 139b6bb
Showing 1 changed file with 8 additions and 3 deletions.
diff --git a/backend/data_processing/update_statistics.py b/backend/data_processing/update_statistics.py
@@ -32,10 +32,15 @@ async def statistics_update():
         raise e
 
     # gets all updated user statistics
-    tasks = [
-        process_user_statistics_update(user, movie_data) for user in statistics_users
+    batch_size = 20
+    batches = [
+        statistics_users[i : i + batch_size]
+        for i in range(0, len(statistics_users), batch_size)
     ]
-    results = await asyncio.gather(*tasks)
+    results = []
+    for batch in batches:
+        tasks = [process_user_statistics_update(user, movie_data) for user in batch]
+        results.extend(await asyncio.gather(*tasks))
     all_stats = {user: stats for user, stats in results if stats is not None}
 
     # updates user statistics in database