fix: check if job is still alive

this fixes a bug where jobs got just stuck in the running queue and never got released

src/Job.ts

@@ -180,6 +180,18 @@ export class Job<DATA = any | void> {
 		return this.agenda.cancel({ _id: this.attrs._id });
 	}
 
+	isDead(): boolean {
+		const definition = this.agenda.definitions[this.attrs.name];
+		const lockDeadline = new Date(Date.now() - definition.lockLifetime);
+
+		// This means a job has "expired", as in it has not been "touched" within the lockoutTime
+		// Remove from local lock
+		if (this.attrs.lockedAt && this.attrs.lockedAt < lockDeadline) {
+			return true;
+		}
+		return false;
+	}
+
 	async touch(progress?: number): Promise<void> {
 		this.attrs.lockedAt = new Date();
 		this.attrs.progress = progress;

src/JobProcessor.ts

@@ -256,7 +256,9 @@ export class JobProcessor {
 		definition: IJobDefinition
 	): Promise<Job | undefined> {
 		const lockDeadline = new Date(Date.now().valueOf() - definition.lockLifetime);
-		log.extend('findAndLockNextJob')('findAndLockNextJob(%s, [Function])', jobName);
+		log.extend('findAndLockNextJob')(
+			`looking for lockable jobs for ${jobName} (lock dead line = ${lockDeadline})`
+		);
 
 		// Find ONE and ONLY ONE job and set the 'lockedAt' time so that job begins to be processed
 		const result = await this.agenda.db.getNextJobToRun(jobName, this.nextScanAt, lockDeadline);
@@ -325,6 +327,8 @@ export class JobProcessor {
 				this.enqueueJob(job);
 				await this.jobQueueFilling(name);
 				// this.jobProcessing();
+			} else {
+				log.extend('jobQueueFilling')('Cannot lock job [%s]', name);
 			}
 		} catch (error) {
 			log.extend('jobQueueFilling')('[%s] job lock failed while filling queue', name, error);
@@ -400,13 +404,9 @@ export class JobProcessor {
 			(!jobDefinition.concurrency || !status || status.running < jobDefinition.concurrency) &&
 			this.runningJobs.length < this.maxConcurrency
 		) {
-			// Get the deadline of when the job is not supposed to go past for locking
-			const lockDeadline = new Date(Date.now() - jobDefinition.lockLifetime);
-
-			// This means a job has "expired", as in it has not been "touched" within the lockoutTime
-			// Remove from local lock
 			// NOTE: Shouldn't we update the 'lockedAt' value in MongoDB so it can be picked up on restart?
-			if (job.attrs.lockedAt && job.attrs.lockedAt < lockDeadline) {
+			// -> not needed, as the timeout has been reached, and it will be picked up anyways again
+			if (job.isDead()) {
 				log.extend('runOrRetry')(
 					'[%s:%s] job lock has expired, freeing it up',
 					job.attrs.name,
@@ -435,8 +435,30 @@ export class JobProcessor {
 
 			try {
 				log.extend('runOrRetry')('[%s:%s] processing job', job.attrs.name, job.attrs._id);
+
+				// check if the job is still alive
+				const checkIfJobIsDead = () => {
+					// check every processInterval
+					return new Promise((resolve, reject) =>
+						setTimeout(() => {
+							if (job.isDead()) {
+								reject(
+									new Error(
+										`execution of '${job.attrs.name}' canceled, execution took more than ${
+											this.agenda.definitions[job.attrs.name].lockLifetime
+										}ms. Call touch() for long running jobs to keep them alive.`
+									)
+								);
+								return;
+							}
+							resolve(checkIfJobIsDead());
+						}, this.processEvery)
+					);
+				};
+
 				// CALL THE ACTUAL METHOD TO PROCESS THE JOB!!!
-				await job.run();
+				await Promise.race([job.run(), checkIfJobIsDead()]);
+
 				log.extend('runOrRetry')(
 					'[%s:%s] processing job successfull',
 					job.attrs.name,

test/jobprocessor.test.ts

@@ -1,4 +1,4 @@
-import * as expect from 'expect.js';
+import { expect } from 'chai';
 
 import { Db } from 'mongodb';
 import { Agenda } from '../src';
@@ -15,7 +15,7 @@ const clearJobs = async () => {
 	}
 };
 
-describe('Agenda', function () {
+describe('JobProcessor', function () {
 	// this.timeout(1000000);
 
 	beforeEach(async () => {
@@ -48,35 +48,90 @@ describe('Agenda', function () {
 		await clearJobs();
 	});
 
-	describe('configuration methods', () => {
-		it('ensure new jobs are always filling up running queue', async () => {
-			let shortOneFinished = false;
+	it('ensure new jobs are always filling up running queue', async () => {
+		let shortOneFinished = false;
 
-			agenda.define('test long', async () => {
+		agenda.define('test long', async () => {
+			await new Promise(resolve => setTimeout(resolve, 1000));
+		});
+		agenda.define('test short', async () => {
+			shortOneFinished = true;
+			await new Promise(resolve => setTimeout(resolve, 5));
+		});
+
+		await agenda.start();
+
+		// queue up long ones
+		for (let i = 0; i < 100; i++) {
+			agenda.now('test long');
+		}
+
+		await new Promise(resolve => setTimeout(resolve, 1000));
+
+		// queue more short ones (they should complete first!)
+		for (let j = 0; j < 100; j++) {
+			agenda.now('test short');
+		}
+
+		await new Promise(resolve => setTimeout(resolve, 1000));
+
+		expect(shortOneFinished).to.be.true;
+	});
+
+	it('ensure slow jobs time out', async () => {
+		agenda.define(
+			'test long',
+			async () => {
 				await new Promise(resolve => setTimeout(resolve, 1000));
+			},
+			{ lockLifetime: 500 }
+		);
+
+		await agenda.start();
+
+		// queue up long ones
+		agenda.now('test long');
+
+		const promiseResult = await new Promise(resolve => {
+			agenda.on('error', err => {
+				resolve(err);
 			});
-			agenda.define('test short', async () => {
-				shortOneFinished = true;
-				await new Promise(resolve => setTimeout(resolve, 5));
+
+			agenda.on('success', () => {
+				resolve();
 			});
+		});
 
-			await agenda.start();
+		expect(promiseResult).to.be.an('error');
+	});
 
-			// queue up long ones
-			for (let i = 0; i < 100; i++) {
-				agenda.now('test long');
-			}
+	it('ensure slow jobs do not time out when calling touch', async () => {
+		agenda.define(
+			'test long',
+			async job => {
+				for (let i = 0; i < 10; i++) {
+					await new Promise(resolve => setTimeout(resolve, 100));
+					await job.touch();
+				}
+			},
+			{ lockLifetime: 500 }
+		);
 
-			await new Promise(resolve => setTimeout(resolve, 1000));
+		await agenda.start();
 
-			// queue more short ones (they should complete first!)
-			for (let j = 0; j < 100; j++) {
-				agenda.now('test short');
-			}
+		// queue up long ones
+		agenda.now('test long');
 
-			await new Promise(resolve => setTimeout(resolve, 1000));
+		const promiseResult = await new Promise(resolve => {
+			agenda.on('error', err => {
+				resolve(err);
+			});
 
-			expect(shortOneFinished).to.be(true);
+			agenda.on('success', () => {
+				resolve();
+			});
 		});
+
+		expect(promiseResult).to.not.be.an('error');
 	});
 });