Prevent in-process background services from crashing at setup #381

POPWorldMedia · Dec 8, 2024 · 5d3e5b6 · 5d3e5b6
1 parent f981fae
commit 5d3e5b6
Show file tree

Hide file tree

Showing 7 changed files with 167 additions and 34 deletions.
diff --git a/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/AwardCalculatorJob.cs b/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/AwardCalculatorJob.cs
@@ -3,18 +3,43 @@
 
 namespace PopForums.Mvc.Areas.Forums.BackgroundJobs;
 
-public class AwardCalculatorJob(ISettingsManager settingsManager, IServiceHeartbeatService serviceHeartbeatService, IAwardCalculatorWorker awardCalculatorWorker) : BackgroundService
+public class AwardCalculatorJob(ISettingsManager settingsManager, IServiceHeartbeatService serviceHeartbeatService, IAwardCalculatorWorker awardCalculatorWorker, IServiceProvider serviceProvider) : BackgroundService
 {
-	private readonly PeriodicTimer _timer = new(TimeSpan.FromMilliseconds(settingsManager.Current.ScoringGameCalculatorInterval));
-
 	protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 	{
+		PeriodicTimer timer;
+		try
+		{
+			timer = new(TimeSpan.FromMilliseconds(settingsManager.Current.ScoringGameCalculatorInterval));
+		}
+		catch(Exception ex)
+		{
+			var logger = await GetLogger();
+			logger.LogError(ex, $"Error while executing {GetType().FullName} background job. This job will not restart without restarting the app.");
+			return;
+		}
+
 		while (!stoppingToken.IsCancellationRequested)
 		{
-			awardCalculatorWorker.Execute();
-			await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
-			_timer.Period = TimeSpan.FromMilliseconds(settingsManager.Current.ScoringGameCalculatorInterval);
-			await _timer.WaitForNextTickAsync(stoppingToken);
+			try
+			{
+				awardCalculatorWorker.Execute();
+				await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+				timer.Period = TimeSpan.FromMilliseconds(settingsManager.Current.ScoringGameCalculatorInterval);
+			}
+			catch (Exception ex)
+			{
+				var logger = await GetLogger();
+				logger.LogError(ex, $"Error while executing {GetType().FullName} background job.");
+			}
+			await timer.WaitForNextTickAsync(stoppingToken);
 		}
 	}
+
+	private async Task<ILogger<AwardCalculatorJob>> GetLogger()
+	{
+		await using var scope = serviceProvider.CreateAsyncScope();
+		var logger = scope.ServiceProvider.GetRequiredService<ILogger<AwardCalculatorJob>>();
+		return logger;
+	}
 }
diff --git a/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/CloseAgedTopicsJob.cs b/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/CloseAgedTopicsJob.cs
@@ -3,7 +3,7 @@
 
 namespace PopForums.Mvc.Areas.Forums.BackgroundJobs;
 
-public class CloseAgedTopicsJob(IServiceHeartbeatService serviceHeartbeatService, ICloseAgedTopicsWorker closeAgedTopicsWorker) : BackgroundService
+public class CloseAgedTopicsJob(IServiceHeartbeatService serviceHeartbeatService, ICloseAgedTopicsWorker closeAgedTopicsWorker, IServiceProvider serviceProvider) : BackgroundService
 {
 	private const int IntervalValue = 12;
 	private readonly PeriodicTimer _timer = new(TimeSpan.FromHours(IntervalValue));
@@ -12,9 +12,24 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 	{
 		while (!stoppingToken.IsCancellationRequested)
 		{
-			closeAgedTopicsWorker.Execute();
-			await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+			try
+			{
+				closeAgedTopicsWorker.Execute();
+				await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+			}
+			catch (Exception ex)
+			{
+				var logger = await GetLogger();
+				logger.LogError(ex, $"Error while executing {GetType().FullName} background job.");
+			}
 			await _timer.WaitForNextTickAsync(stoppingToken);
 		}
 	}
+
+	private async Task<ILogger<CloseAgedTopicsJob>> GetLogger()
+	{
+		await using var scope = serviceProvider.CreateAsyncScope();
+		var logger = scope.ServiceProvider.GetRequiredService<ILogger<CloseAgedTopicsJob>>();
+		return logger;
+	}
 }
diff --git a/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/EmailJob.cs b/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/EmailJob.cs
@@ -3,21 +3,45 @@
 
 namespace PopForums.Mvc.Areas.Forums.BackgroundJobs;
 
-public class EmailJob(ISettingsManager settingsManager, IServiceHeartbeatService serviceHeartbeatService, IEmailWorker emailWorker) : BackgroundService
+public class EmailJob(ISettingsManager settingsManager, IServiceHeartbeatService serviceHeartbeatService, IEmailWorker emailWorker, IServiceProvider serviceProvider) : BackgroundService
 {
-	private readonly PeriodicTimer _timer = new(TimeSpan.FromMilliseconds(settingsManager.Current.MailSendingInverval));
-
 	protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 	{
+		PeriodicTimer timer;
+		try
+		{
+			timer = new(TimeSpan.FromMilliseconds(settingsManager.Current.MailSendingInverval));
+		}
+		catch(Exception ex)
+		{
+			var logger = await GetLogger();
+			logger.LogError(ex, $"Error while executing {GetType().FullName} background job. This job will not restart without restarting the app.");
+			return;
+		}
 		while (!stoppingToken.IsCancellationRequested)
 		{
+			try
+			{
 #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
-			emailWorker.Execute();
+				emailWorker.Execute();
 #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
-			await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
-			var newTimeSpan = TimeSpan.FromMilliseconds(settingsManager.Current.MailSendingInverval);
-			_timer.Period = newTimeSpan;
-			await _timer.WaitForNextTickAsync(stoppingToken);
+				await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+				var newTimeSpan = TimeSpan.FromMilliseconds(settingsManager.Current.MailSendingInverval);
+				timer.Period = newTimeSpan;
+			}
+			catch (Exception ex)
+			{
+				var logger = await GetLogger();
+				logger.LogError(ex, $"Error while executing {GetType().FullName} background job.");
+			}
+			await timer.WaitForNextTickAsync(stoppingToken);
 		}
 	}
+
+	private async Task<ILogger<EmailJob>> GetLogger()
+	{
+		await using var scope = serviceProvider.CreateAsyncScope();
+		var logger = scope.ServiceProvider.GetRequiredService<ILogger<EmailJob>>();
+		return logger;
+	}
 }
diff --git a/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/PostImageCleanupJob.cs b/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/PostImageCleanupJob.cs
@@ -3,7 +3,7 @@
 
 namespace PopForums.Mvc.Areas.Forums.BackgroundJobs;
 
-public class PostImageCleanupJob(IServiceHeartbeatService serviceHeartbeatService, IPostImageCleanupWorker postImageCleanupWorker) : BackgroundService
+public class PostImageCleanupJob(IServiceHeartbeatService serviceHeartbeatService, IPostImageCleanupWorker postImageCleanupWorker, IServiceProvider serviceProvider) : BackgroundService
 {
 	private const double IntervalValue = 12;
 	private readonly PeriodicTimer _timer = new(TimeSpan.FromHours(IntervalValue));
@@ -12,9 +12,24 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 	{
 		while (!stoppingToken.IsCancellationRequested)
 		{
-			postImageCleanupWorker.Execute();
-			await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+			try
+			{
+				postImageCleanupWorker.Execute();
+				await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+			}
+			catch (Exception ex)
+			{
+				var logger = await GetLogger();
+				logger.LogError(ex, $"Error while executing {GetType().FullName} background job.");
+			}
 			await _timer.WaitForNextTickAsync(stoppingToken);
 		}
 	}
+
+	private async Task<ILogger<PostImageCleanupJob>> GetLogger()
+	{
+		await using var scope = serviceProvider.CreateAsyncScope();
+		var logger = scope.ServiceProvider.GetRequiredService<ILogger<PostImageCleanupJob>>();
+		return logger;
+	}
 }
diff --git a/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/SearchIndexJob.cs b/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/SearchIndexJob.cs
@@ -3,18 +3,42 @@
 
 namespace PopForums.Mvc.Areas.Forums.BackgroundJobs;
 
-public class SearchIndexJob(ISettingsManager settingsManager, IServiceHeartbeatService serviceHeartbeatService, ISearchIndexWorker searchIndexWorker) : BackgroundService
+public class SearchIndexJob(ISettingsManager settingsManager, IServiceHeartbeatService serviceHeartbeatService, ISearchIndexWorker searchIndexWorker, IServiceProvider serviceProvider) : BackgroundService
 {
-	private readonly PeriodicTimer _timer = new(TimeSpan.FromMilliseconds(settingsManager.Current.SearchIndexingInterval));
-
 	protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 	{
+		PeriodicTimer timer;
+		try
+		{
+			timer = new(TimeSpan.FromMilliseconds(settingsManager.Current.SearchIndexingInterval));
+		}
+		catch(Exception ex)
+		{
+			var logger = await GetLogger();
+			logger.LogError(ex, $"Error while executing {GetType().FullName} background job. This job will not restart without restarting the app.");
+			return;
+		}
 		while (!stoppingToken.IsCancellationRequested)
 		{
-			searchIndexWorker.Execute();
-			await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
-			_timer.Period = TimeSpan.FromMilliseconds(settingsManager.Current.SearchIndexingInterval);
-			await _timer.WaitForNextTickAsync(stoppingToken);
+			try
+			{
+				searchIndexWorker.Execute();
+				await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+				timer.Period = TimeSpan.FromMilliseconds(settingsManager.Current.SearchIndexingInterval);
+			}
+			catch (Exception ex)
+			{
+				var logger = await GetLogger();
+				logger.LogError(ex, $"Error while executing {GetType().FullName} background job.");
+			}
+			await timer.WaitForNextTickAsync(stoppingToken);
 		}
 	}
+
+	private async Task<ILogger<SearchIndexJob>> GetLogger()
+	{
+		await using var scope = serviceProvider.CreateAsyncScope();
+		var logger = scope.ServiceProvider.GetRequiredService<ILogger<SearchIndexJob>>();
+		return logger;
+	}
 }
diff --git a/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/SubscribeNotificationJob.cs b/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/SubscribeNotificationJob.cs
@@ -3,7 +3,7 @@
 
 namespace PopForums.Mvc.Areas.Forums.BackgroundJobs;
 
-public class SubscribeNotificationJob(IServiceHeartbeatService serviceHeartbeatService, ISubscribeNotificationWorker subscribeNotificationWorker) : BackgroundService
+public class SubscribeNotificationJob(IServiceHeartbeatService serviceHeartbeatService, ISubscribeNotificationWorker subscribeNotificationWorker, IServiceProvider serviceProvider) : BackgroundService
 {
 	private const double IntervalValue = 15;
 	private readonly PeriodicTimer _timer = new(TimeSpan.FromSeconds(IntervalValue));
@@ -12,9 +12,24 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 	{
 		while (!stoppingToken.IsCancellationRequested)
 		{
-			subscribeNotificationWorker.Execute();
-			await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+			try
+			{
+				subscribeNotificationWorker.Execute();
+				await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+			}
+			catch (Exception ex)
+			{
+				var logger = await GetLogger();
+				logger.LogError(ex, $"Error while executing {GetType().FullName} background job.");
+			}
 			await _timer.WaitForNextTickAsync(stoppingToken);
 		}
 	}
+
+	private async Task<ILogger<PostImageCleanupJob>> GetLogger()
+	{
+		await using var scope = serviceProvider.CreateAsyncScope();
+		var logger = scope.ServiceProvider.GetRequiredService<ILogger<PostImageCleanupJob>>();
+		return logger;
+	}
 }
diff --git a/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/UserSessionJob.cs b/src/PopForums.Mvc/Areas/Forums/BackgroundJobs/UserSessionJob.cs
@@ -3,7 +3,7 @@
 
 namespace PopForums.Mvc.Areas.Forums.BackgroundJobs;
 
-public class UserSessionJob(IServiceHeartbeatService serviceHeartbeatService, IUserSessionWorker userSessionWorker) : BackgroundService
+public class UserSessionJob(IServiceHeartbeatService serviceHeartbeatService, IUserSessionWorker userSessionWorker, IServiceProvider serviceProvider) : BackgroundService
 {
 	private const double IntervalValue = 1;
 	private readonly PeriodicTimer _timer = new(TimeSpan.FromMinutes(IntervalValue));
@@ -12,9 +12,24 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 	{
 		while (!stoppingToken.IsCancellationRequested)
 		{
-			userSessionWorker.Execute();
-			await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+			try
+			{
+				userSessionWorker.Execute();
+				await serviceHeartbeatService.RecordHeartbeat(GetType().FullName, Environment.MachineName);
+			}
+			catch (Exception ex)
+			{
+				var logger = await GetLogger();
+				logger.LogError(ex, $"Error while executing {GetType().FullName} background job.");
+			}
 			await _timer.WaitForNextTickAsync(stoppingToken);
 		}
 	}
+
+	private async Task<ILogger<UserSessionJob>> GetLogger()
+	{
+		await using var scope = serviceProvider.CreateAsyncScope();
+		var logger = scope.ServiceProvider.GetRequiredService<ILogger<UserSessionJob>>();
+		return logger;
+	}
 }