create container/network with label, best effort clean them up.

microsoft · Nov 7, 2018 · eecfe94 · eecfe94
1 parent ca5df34
commit eecfe94
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 3 deletions.
diff --git a/src/Agent.Worker/Container/DockerCommandManager.cs b/src/Agent.Worker/Container/DockerCommandManager.cs
@@ -23,20 +23,25 @@ public interface IDockerCommandManager : IAgentService
         Task<int> DockerLogs(IExecutionContext context, string containerId);
         Task<List<string>> DockerPS(IExecutionContext context, string containerId, string filter);
         Task<int> DockerRemove(IExecutionContext context, string containerId);
+        Task<int> DockerContainerPrune(IExecutionContext context);
         Task<int> DockerNetworkCreate(IExecutionContext context, string network);
         Task<int> DockerNetworkRemove(IExecutionContext context, string network);
+        Task<int> DockerNetworkPrune(IExecutionContext context);
         Task<int> DockerExec(IExecutionContext context, string containerId, string options, string command);
         Task<int> DockerExec(IExecutionContext context, string containerId, string options, string command, List<string> outputs);
     }
 
     public class DockerCommandManager : AgentService, IDockerCommandManager
     {
+        private string _agentInstanceLabel;
+
         public string DockerPath { get; private set; }
 
         public override void Initialize(IHostContext hostContext)
         {
             base.Initialize(hostContext);
             DockerPath = WhichUtil.Which("docker", true, Trace);
+            _agentInstanceLabel = IOUtil.GetPathHash(hostContext.GetDirectory(WellKnownDirectory.Bin)).Substring(0, 5);
         }
 
         public async Task<DockerVersion> DockerVersion(IExecutionContext context)
@@ -123,9 +128,9 @@ public async Task<string> DockerCreate(IExecutionContext context, string display
             string node = context.Container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
             string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
 #if OS_WINDOWS
-            string dockerArgs = $"--name {displayName} {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}";  // add --network={network} and -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09)
+            string dockerArgs = $"--name {displayName} --label {_agentInstanceLabel} {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}";  // add --network={network} and -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09)
 #else
-            string dockerArgs = $"--name {displayName} --network={network} -v /var/run/docker.sock:/var/run/docker.sock {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}";
+            string dockerArgs = $"--name {displayName} --network={network} --label {_agentInstanceLabel} -v /var/run/docker.sock:/var/run/docker.sock {options} {dockerEnvArgs} {dockerMountVolumesArgs} {image} {sleepCommand}";
 #endif
             List<string> outputStrings = await ExecuteDockerCommandAsync(context, "create", dockerArgs);
             return outputStrings.FirstOrDefault();
@@ -146,6 +151,11 @@ public async Task<int> DockerRemove(IExecutionContext context, string containerI
             return await ExecuteDockerCommandAsync(context, "rm", containerId, context.CancellationToken);
         }
 
+        public async Task<int> DockerContainerPrune(IExecutionContext context)
+        {
+            return await ExecuteDockerCommandAsync(context, "container", $"prune --force --filter \"label={_agentInstanceLabel}\"", context.CancellationToken);
+        }
+
         public async Task<int> DockerLogs(IExecutionContext context, string containerId)
         {
             return await ExecuteDockerCommandAsync(context, "logs", $"--details {containerId}", context.CancellationToken);
@@ -158,14 +168,19 @@ public async Task<List<string>> DockerPS(IExecutionContext context, string conta
 
         public async Task<int> DockerNetworkCreate(IExecutionContext context, string network)
         {
-            return await ExecuteDockerCommandAsync(context, "network", $"create {network}", context.CancellationToken);
+            return await ExecuteDockerCommandAsync(context, "network", $"create --label {_agentInstanceLabel} {network}", context.CancellationToken);
         }
 
         public async Task<int> DockerNetworkRemove(IExecutionContext context, string network)
         {
             return await ExecuteDockerCommandAsync(context, "network", $"rm {network}", context.CancellationToken);
         }
 
+        public async Task<int> DockerNetworkPrune(IExecutionContext context)
+        {
+            return await ExecuteDockerCommandAsync(context, "network", $"prune --force --filter \"label={_agentInstanceLabel}\"", context.CancellationToken);
+        }
+
         public async Task<int> DockerExec(IExecutionContext context, string containerId, string options, string command)
         {
             return await ExecuteDockerCommandAsync(context, "exec", $"{options} {containerId} {command}", context.CancellationToken);

diff --git a/src/Agent.Worker/ContainerOperationProvider.cs b/src/Agent.Worker/ContainerOperationProvider.cs
@@ -105,6 +105,23 @@ public async Task StartContainerAsync(IExecutionContext executionContext, object
                 throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion));
             }
 
+            // Clean up containers left by previous runs
+            executionContext.Debug($"Delete stale containers from previous jobs");
+            int containerPruneExitCode = await _dockerManger.DockerContainerPrune(executionContext);
+            if (containerPruneExitCode != 0)
+            {
+                executionContext.Warning($"Delete stale containers failed, docker container prune fail with exit code {containerPruneExitCode}");
+            }
+
+#if !OS_WINDOWS
+            executionContext.Debug($"Delete stale container networks from previous jobs");
+            int networkPruneExitCode = await _dockerManger.DockerNetworkPrune(executionContext);
+            if (networkPruneExitCode != 0)
+            {
+                executionContext.Warning($"Delete stale container networks failed, docker network prune fail with exit code {networkPruneExitCode}");
+            }
+#endif
+
             // Login to private docker registry
             string registryServer = string.Empty;
             if (container.ContainerRegistryEndpoint != Guid.Empty)