diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs b/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs index ef87f9b9d..2ed544ba2 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs @@ -30,7 +30,7 @@ public abstract class TentacleBuilder : ITentacleBuilder protected string TentacleThumbprint = Certificates.TentaclePublicThumbprint; bool installAsService = false; - readonly Regex listeningPortRegex = new (@"listen:\/\/.+:(\d+)\/"); + static readonly Regex ListeningPortRegex = new (@"listen:\/\/.+:(\d+)\/"); readonly Dictionary runTentacleEnvironmentVariables = new(); TemporaryDirectory? homeDirectory; @@ -98,10 +98,12 @@ protected async Task StartTentacle( ILogger logger, CancellationToken cancellationToken) { + var log = new SerilogLoggerBuilder().Build(); + var runningTentacle = new RunningTentacle( new FileInfo(tentacleExe), tempDirectory, - startTentacleFunction: ct => RunTentacle(serviceUri, tentacleExe, instanceName, tempDirectory, ct), + startTentacleFunction: ct => RunTentacle(serviceUri, tentacleExe, instanceName, tempDirectory, log, ct), tentacleThumbprint, instanceName, HomeDirectory.DirectoryPath, @@ -122,7 +124,7 @@ protected async Task StartTentacle( } catch (Exception e) { - new SerilogLoggerBuilder().Build().Information(e, "Error disposing tentacle after tentacle failed to start."); + log.Information(e, "Error disposing tentacle after tentacle failed to start."); } throw; @@ -134,12 +136,12 @@ protected async Task StartTentacle( string tentacleExe, string instanceName, TemporaryDirectory tempDirectory, + ILogger log, CancellationToken cancellationToken) { var hasTentacleStarted = new ManualResetEventSlim(); hasTentacleStarted.Reset(); int? listeningPort = null; - var lastLogContext = string.Empty; var runningTentacle = Task.Run(async () => { @@ -188,29 +190,58 @@ await RunTentacleCommandOutOfProcess( throw new Exception("Failed to start service"); } - while (listeningPort == null && !cancellationToken.IsCancellationRequested) - { - var logFilePath = Path.Combine(tempDirectory.DirectoryPath, "Logs", "OctopusTentacle.txt"); + using var timeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationTokenSource.Token); + + log.Information("Waiting for the Tentacle Service to start up and assign a Listening Port / no port if Polling"); + var tentacleState = await WaitForTentacleToStart(tempDirectory, linkedCancellationTokenSource.Token); - await using (var stream = new FileStream(logFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - using (var reader = new StreamReader(stream)) - { - var logContent = await reader.ReadToEndAsync(); - lastLogContext = logContent; - } + tentacleState.Started = false; - if (lastLogContext.Contains("Listener started")) + if (tentacleState.Started) + { + listeningPort = tentacleState.ListeningPort; + hasTentacleStarted.Set(); + } + else + { + log.Warning("The Tentacle failed to start correctly. Trying Again. Last Log File Content"); + log.Warning(tentacleState.LogContent); + + File.Delete(Path.Combine(tempDirectory.DirectoryPath, "Logs", "OctopusTentacle.txt")); + + await RunTentacleCommandOutOfProcess( + tentacleExe, + new[] { "service", "--stop", $"--instance={instanceName}" }, + tempDirectory, + s => + { }, + runTentacleEnvironmentVariables, + cancellationToken); + + File.Delete(Path.Combine(tempDirectory.DirectoryPath, "Logs", "OctopusTentacle.txt")); + + await RunTentacleCommandOutOfProcess( + tentacleExe, + new[] { "service", "--start", $"--instance={instanceName}" }, + tempDirectory, + s => + { }, + runTentacleEnvironmentVariables, + cancellationToken); + + tentacleState = await WaitForTentacleToStart(tempDirectory, cancellationToken); + + if (tentacleState.Started) { - listeningPort = Convert.ToInt32(listeningPortRegex.Match(lastLogContext).Groups[1].Value); + listeningPort = tentacleState.ListeningPort; + hasTentacleStarted.Set(); } - - if (lastLogContext.Contains("Agent will not listen") || lastLogContext.Contains("Agent listening on")) + else { - hasTentacleStarted.Set(); - break; + log.Error("The Tentacle failed to start correctly. Last Log File Content"); + log.Error(tentacleState.LogContent); } - - await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); } } else @@ -223,7 +254,7 @@ await RunTentacleCommandOutOfProcess( { if (s.Contains("Listener started")) { - listeningPort = Convert.ToInt32(listeningPortRegex.Match(s).Groups[1].Value); + listeningPort = Convert.ToInt32(ListeningPortRegex.Match(s).Groups[1].Value); } else if (s.Contains("Agent will not listen") || s.Contains("Agent listening on")) { @@ -241,7 +272,9 @@ await RunTentacleCommandOutOfProcess( } }, cancellationToken); - await Task.WhenAny(runningTentacle, WaitHandleAsyncFactory.FromWaitHandle(hasTentacleStarted.WaitHandle, TimeSpan.FromMinutes(1))); + using var cleanupCancellationTokenSource = new CancellationTokenSource(); + await Task.WhenAny(runningTentacle, Task.Delay(TimeSpan.FromMinutes(1), cleanupCancellationTokenSource.Token)); + cleanupCancellationTokenSource.Cancel(); // Will throw. if (runningTentacle.IsCompleted) @@ -251,8 +284,6 @@ await RunTentacleCommandOutOfProcess( if (!hasTentacleStarted.IsSet) { - new SerilogLoggerBuilder().Build().Information(lastLogContext); - throw new Exception("Tentacle did not appear to start correctly"); } @@ -264,6 +295,38 @@ await RunTentacleCommandOutOfProcess( return (runningTentacle, serviceUri); } + static async Task<(bool Started, int? ListeningPort, string LogContent)> WaitForTentacleToStart(TemporaryDirectory tempDirectory, CancellationToken localCancellationToken) + { + var lastLogFileContents = string.Empty; + int? listeningPort = null; + + while (listeningPort == null && !localCancellationToken.IsCancellationRequested) + { + var logFilePath = Path.Combine(tempDirectory.DirectoryPath, "Logs", "OctopusTentacle.txt"); + + await using (var stream = new FileStream(logFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + using (var reader = new StreamReader(stream)) + { + var logContent = await reader.ReadToEndAsync(); + lastLogFileContents = logContent; + } + + if (lastLogFileContents.Contains("Listener started")) + { + listeningPort = Convert.ToInt32(ListeningPortRegex.Match(lastLogFileContents).Groups[1].Value); + } + + if (lastLogFileContents.Contains("Agent will not listen") || lastLogFileContents.Contains("Agent listening on")) + { + return (true, listeningPort, lastLogFileContents); + } + + await Task.Delay(TimeSpan.FromSeconds(1), CancellationToken.None); + } + + return (false, listeningPort, lastLogFileContents); + } + protected async Task AddCertificateToTentacle(string tentacleExe, string instanceName, string tentaclePfxPath, TemporaryDirectory tmp, CancellationToken cancellationToken) { await RunTentacleCommand(tentacleExe, new[] {"import-certificate", $"--from-file={tentaclePfxPath}", $"--instance={instanceName}"}, tmp, cancellationToken); diff --git a/source/Octopus.Tentacle/Tentacle.exe.nlog b/source/Octopus.Tentacle/Tentacle.exe.nlog index 83e4454ff..e9230dcef 100644 --- a/source/Octopus.Tentacle/Tentacle.exe.nlog +++ b/source/Octopus.Tentacle/Tentacle.exe.nlog @@ -34,7 +34,7 @@ - +