Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "Avastancu/joannaakl/service container error log (#2110)" #2176

Merged
merged 1 commit into from
Oct 4, 2022
Merged

Revert "Avastancu/joannaakl/service container error log (#2110)" #2176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions src/Runner.Worker/Container/ContainerInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,6 @@ public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container,
public bool IsJobContainer { get; set; }
public bool IsAlpine { get; set; }

public bool FailedInitialization { get; set; }

public IDictionary<string, string> ContainerEnvironmentVariables
{
get
Expand Down
59 changes: 19 additions & 40 deletions src/Runner.Worker/ContainerOperationProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,41 +98,12 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec
await StartContainerAsync(executionContext, container);
}

await RunContainersHealthcheck(executionContext, containers);
}

public async Task RunContainersHealthcheck(IExecutionContext executionContext, List<ContainerInfo> containers)
{
executionContext.Output("##[group]Waiting for all services to be ready");

var unhealthyContainers = new List<ContainerInfo>();
foreach (var container in containers.Where(c => !c.IsJobContainer))
{
var healthcheck = await ContainerHealthcheck(executionContext, container);

if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase))
{
unhealthyContainers.Add(container);
}
else
{
executionContext.Output($"{container.ContainerNetworkAlias} service is healthy.");
}
await ContainerHealthcheck(executionContext, container);
}
executionContext.Output("##[endgroup]");

if (unhealthyContainers.Count > 0)
{
foreach (var container in unhealthyContainers)
{
executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed.");
await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId);
executionContext.Error($"Failed to initialize container {container.ContainerImage}");
container.FailedInitialization = true;
executionContext.Output("##[endgroup]");
}
throw new InvalidOperationException("One or more containers failed to start.");
}
}

public async Task StopContainersAsync(IExecutionContext executionContext, object data)
Expand Down Expand Up @@ -328,15 +299,16 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai

if (!string.IsNullOrEmpty(container.ContainerId))
{
if (!container.IsJobContainer && !container.FailedInitialization)
if (!container.IsJobContainer)
{
executionContext.Output($"Print service container logs: {container.ContainerDisplayName}");
// Print logs for service container jobs (not the "action" job itself b/c that's already logged).
executionContext.Output($"Print service container logs: {container.ContainerDisplayName}");

int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
}

executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}");
Expand Down Expand Up @@ -423,14 +395,14 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex
}
}

private async Task<string> ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container)
private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container)
{
string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\"";
string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault();
if (string.IsNullOrEmpty(serviceHealth))
{
// Container has no HEALTHCHECK
return String.Empty;
return;
}
var retryCount = 0;
while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase))
Expand All @@ -441,7 +413,14 @@ private async Task<string> ContainerHealthcheck(IExecutionContext executionConte
serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault();
retryCount++;
}
return serviceHealth;
if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase))
{
executionContext.Output($"{container.ContainerNetworkAlias} service is healthy.");
}
else
{
throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}.");
}
}

private async Task<string> ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container)
Expand Down
108 changes: 0 additions & 108 deletions src/Test/L0/Worker/ContainerOperationProviderL0.cs
View file
Open in desktop

This file was deleted.