Skip to content

Commit

Permalink
HARMONY-1538: Add more logging to debug tests
Browse files Browse the repository at this point in the history
  • Loading branch information
indiejames committed Oct 13, 2023
1 parent a8e3106 commit 0f309d8
Showing 1 changed file with 12 additions and 0 deletions.
12 changes: 12 additions & 0 deletions services/service-runner/app/workers/pull-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,13 @@ async function _pullAndDoWork(repeat = true): Promise<void> {
try {
// remove any previous work items to prevent the pod from running out of disk space
const regex = /^(?!WORKING|TERMINATING)(.+)$/;
logger.info('Starting to empty directory');
await emptyDirectory(env.workingDir, regex);
logger.info('Directory emptied');
// write out the WORKING file to prevent pod termination while working
logger.info('WRITING WORKING FILE');
await fs.writeFile(workingFilePath, '1');
logger.info('WORKING FILE WRITTEN');
} catch (e) {
// We'll continue on even if we have issues cleaning up - it just means the pod may end
// up being evicted at some point due to running out of ephemeral storage space
Expand All @@ -169,12 +173,16 @@ async function _pullAndDoWork(repeat = true): Promise<void> {
// check to see if we are terminating
const terminationFilePath = path.join(env.workingDir, 'TERMINATING');
try {
logger.info('CHECKING FOR TERMINATING FILE');
await fs.access(terminationFilePath);
logger.info('DONE CHECKING FOR TERMINATING FILE');
// TERMINATING file exists so PreStop handler is requesting termination
logger.warn('Received TERMINATION request, no longer processing work');
try {
// Clean up the TERMINATING file to ensure we do not stay in an infinite loop terminating
logger.info('UNLINKNG TERMINATING FILE');
await fs.unlink(terminationFilePath);
logger.info('TERMINATING FILE UNLINKED');
} catch (e) {
logger.error('Error removing TERMINATING file, will still attempt to quit');
logger.error(e);
Expand All @@ -185,6 +193,8 @@ async function _pullAndDoWork(repeat = true): Promise<void> {
// expected if file does not exist
}

logger.info('ABOUT TO POLL FOR WORK');

pullCounter += 1;
logger.debug('Polling for work');
if (pullCounter === pullLogPeriod) {
Expand Down Expand Up @@ -225,13 +235,15 @@ async function _pullAndDoWork(repeat = true): Promise<void> {
logger.error(e);
} finally {
// remove the WORKING file
logger.info('UNLINKING WORKING FILE');
try {
await fs.unlink(workingFilePath);
} catch {
// log this, but don't let it stop things
logger.error('Failed to delete /tmp/WORKING');
}
if (repeat) {
logger.info('REPEATING POLLING FOR WORK');
setTimeout(_pullAndDoWork, pollingInterval);
}
}
Expand Down

0 comments on commit 0f309d8

Please sign in to comment.