From 54d70aa57c80dac78267fe7b6da819bef8f3bb23 Mon Sep 17 00:00:00 2001
From: Andrew Brazzatti <a.brazzatti@qcif.edu.au>
Date: Tue, 22 Oct 2024 02:48:18 +0000
Subject: [PATCH] Added a scheduled agenda job that runs every 5 minutes to
 clean up finished agenda jobs. This should improve performance for the job
 processor.

---
 config/agendaQueue.js                         |  8 +++++++
 typescript/api/services/AgendaQueueService.ts | 23 +++++++++++++++++--
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/config/agendaQueue.js b/config/agendaQueue.js
index acef6cc47c..e8fe05d0e1 100644
--- a/config/agendaQueue.js
+++ b/config/agendaQueue.js
@@ -45,6 +45,14 @@ module.exports.agendaQueue = {
     {
       name: 'RaidMintRetryJob',
       fnName: 'raidservice.mintRetryJob'
+    },
+    {
+      name: 'MoveCompletedJobsToHistory',
+      fnName: 'agendaqueueservice.moveCompletedJobsToHistory',
+      schedule: {
+        method: 'every',
+        intervalOrSchedule: '5 minutes'
+      }
     }
   ]
 };
diff --git a/typescript/api/services/AgendaQueueService.ts b/typescript/api/services/AgendaQueueService.ts
index 241f53a120..0f6f989754 100644
--- a/typescript/api/services/AgendaQueueService.ts
+++ b/typescript/api/services/AgendaQueueService.ts
@@ -60,7 +60,8 @@ export module Services {
        'now',
        'jobs',
        'sampleFunctionToDemonstrateHowToDefineAJobFunction',
-       'defineJob'
+       'defineJob',
+       'moveCompletedJobsToHistory'
       ];
 
       protected agenda: Agenda;
@@ -128,7 +129,7 @@ export module Services {
             if (method == 'now') {
               this.now(job.name, data)
             } else if (method == 'every') {
-              this.every(job.name, intervalOrSchedule, data, opts);
+              this.every(job.name, intervalOrSchedule,  data, opts);
             } else if (method == 'schedule') {
               this.schedule(job.name, intervalOrSchedule, data);
             } else {
@@ -185,6 +186,24 @@ export module Services {
 
       }
 
+      /**
+       * 
+       * There are significant slowdowns with Agenda when the jobs collection grows large. This function moves all completed jobs to a history collection so we still have the data but it doesn't affect the peformance of the job processor. 
+       * 
+       * @param job 
+       */
+      public async moveCompletedJobsToHistory(job:any) {
+        const dbManager = User.getDatastore().manager;
+        const collectionName = _.get(sails.config.agendaQueue, 'collection', 'agendaJobs');
+        await dbManager.collection(collectionName).find({ nextRunAt: null }).forEach(async (doc) => {
+          await dbManager.collection(`${collectionName}History`).insertOne(doc);
+          await dbManager.collection(collectionName).deleteOne({ _id: doc._id });
+        });
+
+        sails.log.verbose(`moveCompletedJobsToHistory:: Moved completed jobs to history`);
+      }
+
+
       private setOptionIfDefined(agendaOpts, optionName, optionVal) {
         if (!_.isEmpty(optionVal)) {
           _.set(agendaOpts, optionName, optionVal);