diff --git a/src/Agent.Worker/Container/DockerCommandManager.cs b/src/Agent.Worker/Container/DockerCommandManager.cs index 41dc7f9e32..f8f9807d53 100644 --- a/src/Agent.Worker/Container/DockerCommandManager.cs +++ b/src/Agent.Worker/Container/DockerCommandManager.cs @@ -13,18 +13,19 @@ namespace Microsoft.VisualStudio.Services.Agent.Worker.Container public interface IDockerCommandManager : IAgentService { string DockerPath { get; } + string DockerInstanceLabel { get; } Task DockerVersion(IExecutionContext context); Task DockerLogin(IExecutionContext context, string server, string username, string password); Task DockerLogout(IExecutionContext context, string server); Task DockerPull(IExecutionContext context, string image); Task DockerCreate(IExecutionContext context, string displayName, string image, List mountVolumes, string network, string options, IDictionary environment); Task DockerStart(IExecutionContext context, string containerId); - Task DockerStop(IExecutionContext context, string containerId); Task DockerLogs(IExecutionContext context, string containerId); - Task> DockerPS(IExecutionContext context, string containerId, string filter); + Task> DockerPS(IExecutionContext context, string options); Task DockerRemove(IExecutionContext context, string containerId); Task DockerNetworkCreate(IExecutionContext context, string network); Task DockerNetworkRemove(IExecutionContext context, string network); + Task DockerNetworkPrune(IExecutionContext context); Task DockerExec(IExecutionContext context, string containerId, string options, string command); Task DockerExec(IExecutionContext context, string containerId, string options, string command, List outputs); } @@ -33,10 +34,13 @@ public class DockerCommandManager : AgentService, IDockerCommandManager { public string DockerPath { get; private set; } + public string DockerInstanceLabel { get; private set; } + public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); DockerPath = WhichUtil.Which("docker", true, Trace); + DockerInstanceLabel = IOUtil.GetPathHash(hostContext.GetDirectory(WellKnownDirectory.Bin)).Substring(0, 5); } public async Task DockerVersion(IExecutionContext context) @@ -123,9 +127,9 @@ public async Task DockerCreate(IExecutionContext context, string display string node = context.Container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}")); string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\""; #if OS_WINDOWS - string dockerArgs = $"--name {displayName} {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}"; // add --network={network} and -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09) + string dockerArgs = $"--name {displayName} --label {DockerInstanceLabel} {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}"; // add --network={network} and -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09) #else - string dockerArgs = $"--name {displayName} --network={network} -v /var/run/docker.sock:/var/run/docker.sock {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}"; + string dockerArgs = $"--name {displayName} --network={network} --label {DockerInstanceLabel} -v /var/run/docker.sock:/var/run/docker.sock {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}"; #endif List outputStrings = await ExecuteDockerCommandAsync(context, "create", dockerArgs); return outputStrings.FirstOrDefault(); @@ -136,14 +140,9 @@ public async Task DockerStart(IExecutionContext context, string containerId return await ExecuteDockerCommandAsync(context, "start", containerId, context.CancellationToken); } - public async Task DockerStop(IExecutionContext context, string containerId) - { - return await ExecuteDockerCommandAsync(context, "stop", containerId, context.CancellationToken); - } - public async Task DockerRemove(IExecutionContext context, string containerId) { - return await ExecuteDockerCommandAsync(context, "rm", containerId, context.CancellationToken); + return await ExecuteDockerCommandAsync(context, "rm", $"--force {containerId}", context.CancellationToken); } public async Task DockerLogs(IExecutionContext context, string containerId) @@ -151,14 +150,14 @@ public async Task DockerLogs(IExecutionContext context, string containerId) return await ExecuteDockerCommandAsync(context, "logs", $"--details {containerId}", context.CancellationToken); } - public async Task> DockerPS(IExecutionContext context, string containerId, string filter) + public async Task> DockerPS(IExecutionContext context, string options) { - return await ExecuteDockerCommandAsync(context, "ps", $"--all --filter id={containerId} {filter} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); + return await ExecuteDockerCommandAsync(context, "ps", options); } public async Task DockerNetworkCreate(IExecutionContext context, string network) { - return await ExecuteDockerCommandAsync(context, "network", $"create {network}", context.CancellationToken); + return await ExecuteDockerCommandAsync(context, "network", $"create --label {DockerInstanceLabel} {network}", context.CancellationToken); } public async Task DockerNetworkRemove(IExecutionContext context, string network) @@ -166,6 +165,11 @@ public async Task DockerNetworkRemove(IExecutionContext context, string net return await ExecuteDockerCommandAsync(context, "network", $"rm {network}", context.CancellationToken); } + public async Task DockerNetworkPrune(IExecutionContext context) + { + return await ExecuteDockerCommandAsync(context, "network", $"prune --force --filter \"label={DockerInstanceLabel}\"", context.CancellationToken); + } + public async Task DockerExec(IExecutionContext context, string containerId, string options, string command) { return await ExecuteDockerCommandAsync(context, "exec", $"{options} {containerId} {command}", context.CancellationToken); diff --git a/src/Agent.Worker/ContainerOperationProvider.cs b/src/Agent.Worker/ContainerOperationProvider.cs index bddba38ce0..05ce606301 100644 --- a/src/Agent.Worker/ContainerOperationProvider.cs +++ b/src/Agent.Worker/ContainerOperationProvider.cs @@ -105,6 +105,27 @@ public async Task StartContainerAsync(IExecutionContext executionContext, object throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion)); } + // Clean up containers left by previous runs + executionContext.Debug($"Delete stale containers from previous jobs"); + var staleContainers = await _dockerManger.DockerPS(executionContext, $"--all --quiet --no-trunc --filter \"label={_dockerManger.DockerInstanceLabel}\""); + foreach (var staleContainer in staleContainers) + { + int containerRemoveExitCode = await _dockerManger.DockerRemove(executionContext, staleContainer); + if (containerRemoveExitCode != 0) + { + executionContext.Warning($"Delete stale containers failed, docker rm fail with exit code {containerRemoveExitCode} for container {staleContainer}"); + } + } + +#if !OS_WINDOWS + executionContext.Debug($"Delete stale container networks from previous jobs"); + int networkPruneExitCode = await _dockerManger.DockerNetworkPrune(executionContext); + if (networkPruneExitCode != 0) + { + executionContext.Warning($"Delete stale container networks failed, docker network prune fail with exit code {networkPruneExitCode}"); + } +#endif + // Login to private docker registry string registryServer = string.Empty; if (container.ContainerRegistryEndpoint != Guid.Empty) @@ -240,17 +261,17 @@ public async Task StartContainerAsync(IExecutionContext executionContext, object #if !OS_WINDOWS // Ensure bash exist in the image - int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"which bash"); + int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\""); if (execWhichBashExitCode != 0) { try { // Make sure container is up and running - var psOutputs = await _dockerManger.DockerPS(executionContext, container.ContainerId, "--filter status=running"); + var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true) { // container is not up and running, pull docker log for this container. - await _dockerManger.DockerPS(executionContext, container.ContainerId, string.Empty); + await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId); if (logsExitCode != 0) { @@ -353,13 +374,7 @@ public async Task StopContainerAsync(IExecutionContext executionContext, object if (!string.IsNullOrEmpty(container.ContainerId)) { - executionContext.Output($"Stop container: {container.ContainerDisplayName}"); - - int stopExitCode = await _dockerManger.DockerStop(executionContext, container.ContainerId); - if (stopExitCode != 0) - { - executionContext.Error($"Docker stop fail with exit code {stopExitCode}"); - } + executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}"); int rmExitCode = await _dockerManger.DockerRemove(executionContext, container.ContainerId); if (rmExitCode != 0)