Skip to content

Commit

Permalink
create container/network with label, best effort clean them up.
Browse files Browse the repository at this point in the history
  • Loading branch information
@tingluohuang-test @TingluoHuang
tingluohuang-test authored and TingluoHuang committed Nov 13, 2018
1 parent 438b09a commit 866fdd1
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 23 deletions.
30 changes: 17 additions & 13 deletions src/Agent.Worker/Container/DockerCommandManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,19 @@ namespace Microsoft.VisualStudio.Services.Agent.Worker.Container
public interface IDockerCommandManager : IAgentService
{
string DockerPath { get; }
string DockerInstanceLabel { get; }
Task<DockerVersion> DockerVersion(IExecutionContext context);
Task<int> DockerLogin(IExecutionContext context, string server, string username, string password);
Task<int> DockerLogout(IExecutionContext context, string server);
Task<int> DockerPull(IExecutionContext context, string image);
Task<string> DockerCreate(IExecutionContext context, string displayName, string image, List<MountVolume> mountVolumes, string network, string options, IDictionary<string, string> environment);
Task<int> DockerStart(IExecutionContext context, string containerId);
Task<int> DockerStop(IExecutionContext context, string containerId);
Task<int> DockerLogs(IExecutionContext context, string containerId);
Task<List<string>> DockerPS(IExecutionContext context, string containerId, string filter);
Task<List<string>> DockerPS(IExecutionContext context, string options);
Task<int> DockerRemove(IExecutionContext context, string containerId);
Task<int> DockerNetworkCreate(IExecutionContext context, string network);
Task<int> DockerNetworkRemove(IExecutionContext context, string network);
Task<int> DockerNetworkPrune(IExecutionContext context);
Task<int> DockerExec(IExecutionContext context, string containerId, string options, string command);
Task<int> DockerExec(IExecutionContext context, string containerId, string options, string command, List<string> outputs);
}
Expand All @@ -33,10 +34,13 @@ public class DockerCommandManager : AgentService, IDockerCommandManager
{
public string DockerPath { get; private set; }

public string DockerInstanceLabel { get; private set; }

public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
DockerPath = WhichUtil.Which("docker", true, Trace);
DockerInstanceLabel = IOUtil.GetPathHash(hostContext.GetDirectory(WellKnownDirectory.Bin)).Substring(0, 5);
}

public async Task<DockerVersion> DockerVersion(IExecutionContext context)
Expand Down Expand Up @@ -123,9 +127,9 @@ public async Task<string> DockerCreate(IExecutionContext context, string display
string node = context.Container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
#if OS_WINDOWS
string dockerArgs = $"--name {displayName} {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}"; // add --network={network} and -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09)
string dockerArgs = $"--name {displayName} --label {DockerInstanceLabel} {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}"; // add --network={network} and -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09)
#else
string dockerArgs = $"--name {displayName} --network={network} -v /var/run/docker.sock:/var/run/docker.sock {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}";
string dockerArgs = $"--name {displayName} --network={network} --label {DockerInstanceLabel} -v /var/run/docker.sock:/var/run/docker.sock {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}";
#endif
List<string> outputStrings = await ExecuteDockerCommandAsync(context, "create", dockerArgs);
return outputStrings.FirstOrDefault();
Expand All @@ -136,36 +140,36 @@ public async Task<int> DockerStart(IExecutionContext context, string containerId
return await ExecuteDockerCommandAsync(context, "start", containerId, context.CancellationToken);
}

public async Task<int> DockerStop(IExecutionContext context, string containerId)
{
return await ExecuteDockerCommandAsync(context, "stop", containerId, context.CancellationToken);
}

public async Task<int> DockerRemove(IExecutionContext context, string containerId)
{
return await ExecuteDockerCommandAsync(context, "rm", containerId, context.CancellationToken);
return await ExecuteDockerCommandAsync(context, "rm", $"--force {containerId}", context.CancellationToken);
}

public async Task<int> DockerLogs(IExecutionContext context, string containerId)
{
return await ExecuteDockerCommandAsync(context, "logs", $"--details {containerId}", context.CancellationToken);
}

public async Task<List<string>> DockerPS(IExecutionContext context, string containerId, string filter)
public async Task<List<string>> DockerPS(IExecutionContext context, string options)
{
return await ExecuteDockerCommandAsync(context, "ps", $"--all --filter id={containerId} {filter} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
return await ExecuteDockerCommandAsync(context, "ps", options);
}

public async Task<int> DockerNetworkCreate(IExecutionContext context, string network)
{
return await ExecuteDockerCommandAsync(context, "network", $"create {network}", context.CancellationToken);
return await ExecuteDockerCommandAsync(context, "network", $"create --label {DockerInstanceLabel} {network}", context.CancellationToken);
}

public async Task<int> DockerNetworkRemove(IExecutionContext context, string network)
{
return await ExecuteDockerCommandAsync(context, "network", $"rm {network}", context.CancellationToken);
}

public async Task<int> DockerNetworkPrune(IExecutionContext context)
{
return await ExecuteDockerCommandAsync(context, "network", $"prune --force --filter \"label={DockerInstanceLabel}\"", context.CancellationToken);
}

public async Task<int> DockerExec(IExecutionContext context, string containerId, string options, string command)
{
return await ExecuteDockerCommandAsync(context, "exec", $"{options} {containerId} {command}", context.CancellationToken);
Expand Down
35 changes: 25 additions & 10 deletions src/Agent.Worker/ContainerOperationProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,27 @@ public async Task StartContainerAsync(IExecutionContext executionContext, object
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion));
}

// Clean up containers left by previous runs
executionContext.Debug($"Delete stale containers from previous jobs");
var staleContainers = await _dockerManger.DockerPS(executionContext, $"--all --quiet --no-trunc --filter \"label={_dockerManger.DockerInstanceLabel}\"");
foreach (var staleContainer in staleContainers)
{
int containerRemoveExitCode = await _dockerManger.DockerRemove(executionContext, staleContainer);
if (containerRemoveExitCode != 0)
{
executionContext.Warning($"Delete stale containers failed, docker rm fail with exit code {containerRemoveExitCode} for container {staleContainer}");
}
}

#if !OS_WINDOWS
executionContext.Debug($"Delete stale container networks from previous jobs");
int networkPruneExitCode = await _dockerManger.DockerNetworkPrune(executionContext);
if (networkPruneExitCode != 0)
{
executionContext.Warning($"Delete stale container networks failed, docker network prune fail with exit code {networkPruneExitCode}");
}
#endif

// Login to private docker registry
string registryServer = string.Empty;
if (container.ContainerRegistryEndpoint != Guid.Empty)
Expand Down Expand Up @@ -240,17 +261,17 @@ public async Task StartContainerAsync(IExecutionContext executionContext, object

#if !OS_WINDOWS
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"which bash");
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\"");
if (execWhichBashExitCode != 0)
{
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, container.ContainerId, "--filter status=running");
var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, container.ContainerId, string.Empty);
await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
Expand Down Expand Up @@ -353,13 +374,7 @@ public async Task StopContainerAsync(IExecutionContext executionContext, object

if (!string.IsNullOrEmpty(container.ContainerId))
{
executionContext.Output($"Stop container: {container.ContainerDisplayName}");

int stopExitCode = await _dockerManger.DockerStop(executionContext, container.ContainerId);
if (stopExitCode != 0)
{
executionContext.Error($"Docker stop fail with exit code {stopExitCode}");
}
executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}");

int rmExitCode = await _dockerManger.DockerRemove(executionContext, container.ContainerId);
if (rmExitCode != 0)
Expand Down

0 comments on commit 866fdd1

Please sign in to comment.