diff --git a/packages/backend/src/job/job.service.ts b/packages/backend/src/job/job.service.ts index cc0c633..3d402ed 100644 --- a/packages/backend/src/job/job.service.ts +++ b/packages/backend/src/job/job.service.ts @@ -65,7 +65,8 @@ export class JobService { { name: 'REDIS_PORT', value: this.configService.getOrThrow('redis.jobPort') }, { name: 'REDIS_PASSWORD', value: this.configService.get('redis.jobPassword') }, { name: 'COMETS_GLOP', value: './lib/comets_glop' } - ] + ], + imagePullPolicy: 'Always' }); this.jobTemplate.spec.template.spec.restartPolicy = 'Never'; this.jobTemplate.spec.backoffLimit = 0; diff --git a/packages/backend/src/simulation/simulation.consumer.ts b/packages/backend/src/simulation/simulation.consumer.ts index b11e195..270f5ba 100644 --- a/packages/backend/src/simulation/simulation.consumer.ts +++ b/packages/backend/src/simulation/simulation.consumer.ts @@ -13,19 +13,14 @@ export class SimulationRequestConsumer extends WorkerHost { async process(job: Job): Promise { const request = job.data; // Start the COMETs job - try { - const jobName = await this.jobService.triggerJob(request); - // Wait for the job to complete - const status = await this.awaitCompletion(jobName); - // If the status is error, collect and send error data - if (status == JobStatus.FAILURE) { - await this.handleError(request, jobName); - } else { - await this.handleSuccess(request, jobName); - } - } catch (e) { - console.error(e); - throw e; + const jobName = await this.jobService.triggerJob(request); + // Wait for the job to complete + const status = await this.awaitCompletion(jobName); + // If the status is error, collect and send error data + if (status == JobStatus.FAILURE) { + await this.handleError(request, jobName); + } else { + await this.handleSuccess(request, jobName); } } @@ -50,14 +45,18 @@ export class SimulationRequestConsumer extends WorkerHost { * Keep checking the status of the job once a second */ private async awaitCompletion(jobName: string): Promise { - return new Promise(async (resolve, _reject) => { + return new Promise(async (resolve, reject) => { const checkOperations = async () => { - const status = await this.jobService.getJobStatus(jobName); - if (status == JobStatus.RUNNING) { - setTimeout(checkOperations, 1000); - return; - } else { - resolve(status); + try { + const status = await this.jobService.getJobStatus(jobName); + if (status == JobStatus.RUNNING) { + setTimeout(checkOperations, 1000); + return; + } else { + resolve(status); + } + } catch (e) { + reject(e); } }; await checkOperations(); diff --git a/packages/runner/.gitignore b/packages/runner/.gitignore index 6543d6c..26241df 100644 --- a/packages/runner/.gitignore +++ b/packages/runner/.gitignore @@ -1,5 +1,5 @@ venv/ -sim_files/ +sim_files/* !sim_files/.gitkeep sample.sh sample-docker.sh diff --git a/packages/runner/sim_files/.gitkeep b/packages/runner/sim_files/.gitkeep new file mode 100644 index 0000000..e69de29