Skip to content

Commit

Permalink
.
Browse files Browse the repository at this point in the history
  • Loading branch information
@nathanwoctopusdeploy
nathanwoctopusdeploy committed Dec 4, 2023
1 parent 13807bf commit a90e757
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 26 deletions.
113 changes: 88 additions & 25 deletions source/Octopus.Tentacle.Tests.Integration/Support/TentacleBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public abstract class TentacleBuilder<T> : ITentacleBuilder
protected string TentacleThumbprint = Certificates.TentaclePublicThumbprint;
bool installAsService = false;

readonly Regex listeningPortRegex = new (@"listen:\/\/.+:(\d+)\/");
static readonly Regex ListeningPortRegex = new (@"listen:\/\/.+:(\d+)\/");
readonly Dictionary<string, string> runTentacleEnvironmentVariables = new();

TemporaryDirectory? homeDirectory;
Expand Down Expand Up @@ -98,10 +98,12 @@ protected async Task<RunningTentacle> StartTentacle(
ILogger logger,
CancellationToken cancellationToken)
{
var log = new SerilogLoggerBuilder().Build();

var runningTentacle = new RunningTentacle(
new FileInfo(tentacleExe),
tempDirectory,
startTentacleFunction: ct => RunTentacle(serviceUri, tentacleExe, instanceName, tempDirectory, ct),
startTentacleFunction: ct => RunTentacle(serviceUri, tentacleExe, instanceName, tempDirectory, log, ct),
tentacleThumbprint,
instanceName,
HomeDirectory.DirectoryPath,
Expand All @@ -122,7 +124,7 @@ protected async Task<RunningTentacle> StartTentacle(
}
catch (Exception e)
{
new SerilogLoggerBuilder().Build().Information(e, "Error disposing tentacle after tentacle failed to start.");
log.Information(e, "Error disposing tentacle after tentacle failed to start.");
}

throw;
Expand All @@ -134,12 +136,12 @@ protected async Task<RunningTentacle> StartTentacle(
string tentacleExe,
string instanceName,
TemporaryDirectory tempDirectory,
ILogger log,
CancellationToken cancellationToken)
{
var hasTentacleStarted = new ManualResetEventSlim();
hasTentacleStarted.Reset();
int? listeningPort = null;
var lastLogContext = string.Empty;

var runningTentacle = Task.Run(async () =>
{
Expand Down Expand Up @@ -188,29 +190,58 @@ await RunTentacleCommandOutOfProcess(
throw new Exception("Failed to start service");
}

while (listeningPort == null && !cancellationToken.IsCancellationRequested)
{
var logFilePath = Path.Combine(tempDirectory.DirectoryPath, "Logs", "OctopusTentacle.txt");
using var timeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(20));
using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationTokenSource.Token);

log.Information("Waiting for the Tentacle Service to start up and assign a Listening Port / no port if Polling");
var tentacleState = await WaitForTentacleToStart(tempDirectory, linkedCancellationTokenSource.Token);

await using (var stream = new FileStream(logFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var reader = new StreamReader(stream))
{
var logContent = await reader.ReadToEndAsync();
lastLogContext = logContent;
}
tentacleState.Started = false;

if (lastLogContext.Contains("Listener started"))
if (tentacleState.Started)
{
listeningPort = tentacleState.ListeningPort;
hasTentacleStarted.Set();
}
else
{
log.Warning("The Tentacle failed to start correctly. Trying Again. Last Log File Content");
log.Warning(tentacleState.LogContent);

File.Delete(Path.Combine(tempDirectory.DirectoryPath, "Logs", "OctopusTentacle.txt"));

await RunTentacleCommandOutOfProcess(
tentacleExe,
new[] { "service", "--stop", $"--instance={instanceName}" },
tempDirectory,
s =>
{ },
runTentacleEnvironmentVariables,
cancellationToken);

File.Delete(Path.Combine(tempDirectory.DirectoryPath, "Logs", "OctopusTentacle.txt"));

await RunTentacleCommandOutOfProcess(
tentacleExe,
new[] { "service", "--start", $"--instance={instanceName}" },
tempDirectory,
s =>
{ },
runTentacleEnvironmentVariables,
cancellationToken);

tentacleState = await WaitForTentacleToStart(tempDirectory, cancellationToken);

if (tentacleState.Started)
{
listeningPort = Convert.ToInt32(listeningPortRegex.Match(lastLogContext).Groups[1].Value);
listeningPort = tentacleState.ListeningPort;
hasTentacleStarted.Set();
}

if (lastLogContext.Contains("Agent will not listen") || lastLogContext.Contains("Agent listening on"))
else
{
hasTentacleStarted.Set();
break;
log.Error("The Tentacle failed to start correctly. Last Log File Content");
log.Error(tentacleState.LogContent);
}

await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken);
}
}
else
Expand All @@ -223,7 +254,7 @@ await RunTentacleCommandOutOfProcess(
{
if (s.Contains("Listener started"))
{
listeningPort = Convert.ToInt32(listeningPortRegex.Match(s).Groups[1].Value);
listeningPort = Convert.ToInt32(ListeningPortRegex.Match(s).Groups[1].Value);
}
else if (s.Contains("Agent will not listen") || s.Contains("Agent listening on"))
{
Expand All @@ -241,7 +272,9 @@ await RunTentacleCommandOutOfProcess(
}
}, cancellationToken);

await Task.WhenAny(runningTentacle, WaitHandleAsyncFactory.FromWaitHandle(hasTentacleStarted.WaitHandle, TimeSpan.FromMinutes(1)));
using var cleanupCancellationTokenSource = new CancellationTokenSource();
await Task.WhenAny(runningTentacle, Task.Delay(TimeSpan.FromMinutes(1), cleanupCancellationTokenSource.Token));
cleanupCancellationTokenSource.Cancel();

// Will throw.
if (runningTentacle.IsCompleted)
Expand All @@ -251,8 +284,6 @@ await RunTentacleCommandOutOfProcess(

if (!hasTentacleStarted.IsSet)
{
new SerilogLoggerBuilder().Build().Information(lastLogContext);

throw new Exception("Tentacle did not appear to start correctly");
}

Expand All @@ -264,6 +295,38 @@ await RunTentacleCommandOutOfProcess(
return (runningTentacle, serviceUri);
}

static async Task<(bool Started, int? ListeningPort, string LogContent)> WaitForTentacleToStart(TemporaryDirectory tempDirectory, CancellationToken localCancellationToken)
{
var lastLogFileContents = string.Empty;
int? listeningPort = null;

while (listeningPort == null && !localCancellationToken.IsCancellationRequested)
{
var logFilePath = Path.Combine(tempDirectory.DirectoryPath, "Logs", "OctopusTentacle.txt");

await using (var stream = new FileStream(logFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var reader = new StreamReader(stream))
{
var logContent = await reader.ReadToEndAsync();
lastLogFileContents = logContent;
}

if (lastLogFileContents.Contains("Listener started"))
{
listeningPort = Convert.ToInt32(ListeningPortRegex.Match(lastLogFileContents).Groups[1].Value);
}

if (lastLogFileContents.Contains("Agent will not listen") || lastLogFileContents.Contains("Agent listening on"))
{
return (true, listeningPort, lastLogFileContents);
}

await Task.Delay(TimeSpan.FromSeconds(1), CancellationToken.None);
}

return (false, listeningPort, lastLogFileContents);
}

protected async Task AddCertificateToTentacle(string tentacleExe, string instanceName, string tentaclePfxPath, TemporaryDirectory tmp, CancellationToken cancellationToken)
{
await RunTentacleCommand(tentacleExe, new[] {"import-certificate", $"--from-file={tentaclePfxPath}", $"--instance={instanceName}"}, tmp, cancellationToken);
Expand Down
2 changes: 1 addition & 1 deletion source/Octopus.Tentacle/Tentacle.exe.nlog
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
<logger name="LogFileOnlyLogger" writeTo="octopus-log-file" final="true" />
<logger name="*" minlevel="Info" maxLevel="Warn" writeTo="stdout" />
<logger name="*" minlevel="Error" writeTo="stderr" />
<logger name="*" minlevel="Info" writeTo="octopus-log-file" />
<logger name="*" minlevel="Trace" writeTo="octopus-log-file" />
<logger name="*" minlevel="Fatal" writeTo="eventlog" />
</rules>
</nlog>

0 comments on commit a90e757

Please sign in to comment.