diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/AllContainersRunningPodWatcher.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/AllContainersRunningPodWatcher.java deleted file mode 100644 index 3432a8d80a..0000000000 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/AllContainersRunningPodWatcher.java +++ /dev/null @@ -1,166 +0,0 @@ -package org.csanchez.jenkins.plugins.kubernetes; - -import static java.util.stream.Collectors.joining; - -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.logging.Level; -import java.util.logging.Logger; - -import io.fabric8.kubernetes.api.model.ContainerStatus; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.PodStatus; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientTimeoutException; -import io.fabric8.kubernetes.client.Watcher; -import io.fabric8.kubernetes.client.WatcherException; -import io.fabric8.kubernetes.client.utils.Serialization; - -/** - * A pod watcher reporting when all containers are running - */ -public class AllContainersRunningPodWatcher implements Watcher { - private static final Logger LOGGER = Logger.getLogger(AllContainersRunningPodWatcher.class.getName()); - - private final CountDownLatch latch = new CountDownLatch(1); - private final AtomicReference reference = new AtomicReference<>(); - - private Pod pod; - - private KubernetesClient client; - - public AllContainersRunningPodWatcher(KubernetesClient client, Pod pod) { - this.client = client; - this.pod = pod; - updateState(pod); - } - - @Override - public void eventReceived(Action action, Pod pod) { - LOGGER.log(Level.FINEST, "[{0}] {1}", new Object[]{action, pod.getMetadata().getName()}); - switch (action) { - case MODIFIED: - updateState(pod); - break; - default: - } - } - - private void updateState(Pod pod) { - if (areAllContainersRunning(pod)) { - LOGGER.log(Level.FINE, "All containers are running for pod {0}", new Object[] {pod.getMetadata().getName()}); - reference.set(pod); - latch.countDown(); - } - } - - boolean areAllContainersRunning(Pod pod) { - return pod.getSpec().getContainers().size() == pod.getStatus().getContainerStatuses().size() && PodUtils.getContainerStatus(pod).stream().allMatch(ContainerStatus::getReady); - } - - @Override - public void onClose(WatcherException cause) { - - } - - /** - * Wait until all pod containers are running - * - * @return the pod - * @throws PodNotRunningException - * if pod or containers are no longer running - * @throws KubernetesClientTimeoutException - * if time ran out - */ - public Pod await(long amount, TimeUnit timeUnit) throws PodNotRunningException { - long started = System.currentTimeMillis(); - long alreadySpent = System.currentTimeMillis() - started; - long remaining = timeUnit.toMillis(amount) - alreadySpent; - if (remaining <= 0) { - return periodicAwait(0, System.currentTimeMillis(), 0, 0); - } - try { - long interval = Math.min(10000L, Math.max(remaining / 10, 1000L)); - long retries = remaining / interval; - return periodicAwait(retries, System.currentTimeMillis(), interval, remaining); - } catch (KubernetesClientTimeoutException e) { - // Wrap using the right timeout - throw new KubernetesClientTimeoutException(pod, amount, timeUnit); - } - } - - private Pod awaitWatcher(long amount, TimeUnit timeUnit) { - try { - if (latch.await(amount, timeUnit)) { - return reference.get(); - } - throw new KubernetesClientTimeoutException(pod, amount, timeUnit); - } catch (InterruptedException e) { - throw new KubernetesClientTimeoutException(pod, amount, timeUnit); - } - } - - /** - * Wait until all pod containers are running - * - * @return the pod - * @throws PodNotRunningException - * if pod or containers are no longer running - * @throws KubernetesClientTimeoutException - * if time ran out - */ - private Pod periodicAwait(long i, long started, long interval, long amount) throws PodNotRunningException { - Pod pod = client.pods().inNamespace(this.pod.getMetadata().getNamespace()) - .withName(this.pod.getMetadata().getName()).get(); - if (pod == null) { - throw new PodNotRunningException(String.format("Pod is no longer available: %s/%s", - this.pod.getMetadata().getNamespace(), this.pod.getMetadata().getName())); - } else { - LOGGER.finest(() -> "Updating pod for " + this.pod.getMetadata().getNamespace() + "/" + this.pod.getMetadata().getName() + " : " + Serialization.asYaml(pod)); - this.pod = pod; - } - List terminatedContainers = PodUtils.getTerminatedContainers(pod); - if (!terminatedContainers.isEmpty()) { - PodNotRunningException x = new PodNotRunningException(String.format("Pod has terminated containers: %s/%s (%s)", - this.pod.getMetadata().getNamespace(), - this.pod.getMetadata().getName(), - terminatedContainers.stream() - .map(ContainerStatus::getName) - .collect(joining(", ") - ))); - String logs = PodUtils.logLastLines(this.pod, client); - if (logs != null) { - x.addSuppressed(new ContainerLogs(logs)); - } - throw x; - } - if (areAllContainersRunning(pod)) { - return pod; - } - try { - return awaitWatcher(interval, TimeUnit.MILLISECONDS); - } catch (KubernetesClientTimeoutException e) { - if (i <= 0) { - throw e; - } - } - - long remaining = (started + amount) - System.currentTimeMillis(); - long next = Math.max(0, Math.min(remaining, interval)); - return periodicAwait(i - 1, started, next, amount); - } - - public PodStatus getPodStatus() { - return this.pod.getStatus(); - } - - /** @see #await */ - public static final class PodNotRunningException extends Exception { - PodNotRunningException(String s) { - super(s); - } - } - -} diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/ContainerLogs.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/ContainerLogs.java deleted file mode 100644 index 998d955a8b..0000000000 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/ContainerLogs.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.csanchez.jenkins.plugins.kubernetes; - -/** - * Exception class used to convey container logs after a failure for information - */ -public class ContainerLogs extends Exception { - public ContainerLogs() { - } - - public ContainerLogs(String message) { - super(message); - } - - public ContainerLogs(String message, Throwable cause) { - super(message, cause); - } - - public ContainerLogs(Throwable cause) { - super(cause); - } - - public ContainerLogs(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java index 98d5f05932..04151f0b57 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java @@ -35,7 +35,13 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.Watch; +import io.fabric8.kubernetes.client.KubernetesClientTimeoutException; +import io.fabric8.kubernetes.client.internal.readiness.Readiness; +import jenkins.metrics.api.Metrics; +import org.apache.commons.lang.StringUtils; +import org.csanchez.jenkins.plugins.kubernetes.pod.retention.Reaper; +import org.kohsuke.stapler.DataBoundConstructor; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -46,10 +52,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import jenkins.metrics.api.Metrics; -import org.apache.commons.lang.StringUtils; -import org.csanchez.jenkins.plugins.kubernetes.pod.retention.Reaper; -import org.kohsuke.stapler.DataBoundConstructor; import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; @@ -62,9 +64,6 @@ public class KubernetesLauncher extends JNLPLauncher { // Report progress every 30 seconds private static final long REPORT_INTERVAL = TimeUnit.SECONDS.toMillis(30L); - @CheckForNull - private transient AllContainersRunningPodWatcher watcher; - private static final Logger LOGGER = Logger.getLogger(KubernetesLauncher.class.getName()); private boolean launched; @@ -166,12 +165,9 @@ else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfill ObjectMeta podMetadata = pod.getMetadata(); template.getWorkspaceVolume().createVolume(client, podMetadata); template.getVolumes().forEach(volume -> volume.createVolume(client, podMetadata)); - watcher = new AllContainersRunningPodWatcher(client, pod); - try (Watch w1 = client.pods().inNamespace(namespace).withName(podName).watch(watcher); - Watch w2 = eventWatch(client, podName, namespace, runListener)) { - assert watcher != null; // assigned 3 lines above - watcher.await(template.getSlaveConnectTimeout(), TimeUnit.SECONDS); - } + + client.pods().inNamespace(namespace).withName(podName).waitUntilReady(template.getSlaveConnectTimeout(), TimeUnit.SECONDS); + LOGGER.log(INFO, () -> "Pod is running: " + cloudName + " " + namespace + "/" + podName); // We need the pod to be running and connected before returning @@ -256,17 +252,7 @@ else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfill Metrics.metricRegistry().counter(MetricNames.PODS_LAUNCHED).inc(); } catch (Throwable ex) { setProblem(ex); - if (ex instanceof AllContainersRunningPodWatcher.PodNotRunningException) { - Throwable[] suppressed = ex.getSuppressed(); - if (suppressed.length > 0 && suppressed[0] instanceof ContainerLogs) { - runListener.getLogger().println("Unable to provision agent " + node.getNodeName() + " :"); - runListener.getLogger().print(suppressed[0].getMessage()); - } - LOGGER.log(Level.WARNING, String.format("Error in provisioning: %s; agent=%s, template=%s", ex.getMessage(), node, template)); - LOGGER.log(Level.FINE, null, ex); - } else { - LOGGER.log(Level.WARNING, String.format("Error in provisioning; agent=%s, template=%s", node, template), ex); - } + LOGGER.log(Level.WARNING, String.format("Error in provisioning; agent=%s, template=%s", node, template), ex); LOGGER.log(Level.FINER, "Removing Jenkins node: {0}", node.getNodeName()); try { node.terminate(); @@ -277,15 +263,6 @@ else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfill } } - private Watch eventWatch(KubernetesClient client, String podName, String namespace, TaskListener runListener) { - try { - return client.v1().events().inNamespace(namespace).withField("involvedObject.name", podName).watch(new TaskListenerEventWatcher(podName, runListener)); - } catch (KubernetesClientException e) { - LOGGER.log(Level.INFO, e, () -> "Cannot watch events on " + namespace + "/" +podName); - } - return () -> {}; - } - private void checkTerminatedContainers(List terminatedContainers, String podId, String namespace, KubernetesSlave slave, KubernetesClient client) { if (!terminatedContainers.isEmpty()) { @@ -330,8 +307,4 @@ public void setProblem(@CheckForNull Throwable problem) { this.problem = problem; } - public AllContainersRunningPodWatcher getWatcher() { - return watcher; - } - } diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorTest.java index cab62a167c..bfd8e24972 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorTest.java @@ -24,39 +24,24 @@ package org.csanchez.jenkins.plugins.kubernetes.pipeline; -import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.*; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.arrayContaining; -import static org.hamcrest.Matchers.isA; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.fabric8.kubernetes.client.http.WebSocketHandshakeException; -import java.io.ByteArrayOutputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; - import hudson.EnvVars; +import hudson.Launcher; +import hudson.Launcher.DummyLauncher; +import hudson.Launcher.ProcStarter; import hudson.model.Computer; +import hudson.model.Node; +import hudson.slaves.DumbSlave; +import hudson.util.StreamTaskListener; +import io.fabric8.kubernetes.api.model.ContainerBuilder; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodBuilder; +import io.fabric8.kubernetes.client.HttpClientAware; +import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.Watch; +import io.fabric8.kubernetes.client.http.WebSocketHandshakeException; import org.apache.commons.io.output.TeeOutputStream; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.StringUtils; -import org.csanchez.jenkins.plugins.kubernetes.AllContainersRunningPodWatcher; import org.csanchez.jenkins.plugins.kubernetes.KubernetesClientProvider; import org.csanchez.jenkins.plugins.kubernetes.KubernetesCloud; import org.csanchez.jenkins.plugins.kubernetes.KubernetesSlave; @@ -65,27 +50,42 @@ import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TestName; import org.junit.rules.Timeout; import org.jvnet.hudson.test.Issue; +import org.jvnet.hudson.test.JenkinsRule; import org.jvnet.hudson.test.LoggerRule; -import hudson.Launcher; -import hudson.Launcher.DummyLauncher; -import hudson.Launcher.ProcStarter; -import hudson.model.Node; -import hudson.slaves.DumbSlave; -import hudson.util.StreamTaskListener; -import io.fabric8.kubernetes.api.model.ContainerBuilder; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.PodBuilder; -import io.fabric8.kubernetes.client.HttpClientAware; -import io.fabric8.kubernetes.client.KubernetesClient; -import org.junit.Ignore; -import org.jvnet.hudson.test.JenkinsRule; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.assumeKubernetes; +import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.deletePods; +import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.getLabels; +import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.setupCloud; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.isA; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * @author Carlos Sanchez @@ -155,11 +155,7 @@ public void configureCloud() throws Exception { .endSpec().build()); System.out.println("Created pod: " + pod.getMetadata().getName()); - AllContainersRunningPodWatcher watcher = new AllContainersRunningPodWatcher(client, pod); - try (Watch w1 = client.pods().withName(podName).watch(watcher);) { - assert watcher != null; // assigned 3 lines above - watcher.await(30, TimeUnit.SECONDS); - } + client.pods().withName(podName).waitUntilReady(30, TimeUnit.SECONDS); PodTemplate template = new PodTemplate(); template.setName(pod.getMetadata().getName()); agent = mock(KubernetesSlave.class); diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorWindowsTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorWindowsTest.java index f1ad81d866..b49b0b7207 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorWindowsTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorWindowsTest.java @@ -33,20 +33,8 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodBuilder; import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.Watch; -import java.io.ByteArrayOutputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; import org.apache.commons.io.output.TeeOutputStream; import org.apache.commons.lang.RandomStringUtils; -import org.csanchez.jenkins.plugins.kubernetes.AllContainersRunningPodWatcher; import org.csanchez.jenkins.plugins.kubernetes.KubernetesClientProvider; import org.csanchez.jenkins.plugins.kubernetes.KubernetesCloud; import org.csanchez.jenkins.plugins.kubernetes.KubernetesSlave; @@ -63,6 +51,17 @@ import org.jvnet.hudson.test.LoggerRule; import org.jvnet.hudson.test.recipes.WithTimeout; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.WINDOWS_1809_BUILD; import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.assumeKubernetes; import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.assumeWindows; @@ -131,11 +130,7 @@ public void configureCloud() throws Exception { .endSpec().build()); System.out.println("Created pod: " + pod.getMetadata().getName()); - AllContainersRunningPodWatcher watcher = new AllContainersRunningPodWatcher(client, pod); - try (Watch w1 = client.pods().withName(podName).watch(watcher);) { - assert watcher != null; // assigned 3 lines above - watcher.await(10, TimeUnit.MINUTES); - } + client.pods().withName(podName).waitUntilReady(10, TimeUnit.MINUTES); PodTemplate template = new PodTemplate(); template.setName(pod.getMetadata().getName()); agent = mock(KubernetesSlave.class); diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java index 506fd2166e..61dd8f92d5 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java @@ -201,7 +201,6 @@ public void runInPod() throws Exception { r.assertBuildStatusSuccess(r.waitForCompletion(b)); r.assertLogContains("container=busybox", b); r.assertLogContains("script file contents: ", b); - r.assertLogContains("Created container jnlp", b); assertFalse("There are pods leftover after test execution, see previous logs", deletePods(cloud.connect(), getLabels(cloud, this, name), true)); assertThat("routine build should not issue warnings", @@ -244,7 +243,6 @@ public void runIn2Pods() throws Exception { SemaphoreStep.success("pod2/1", null); r.assertBuildStatusSuccess(r.waitForCompletion(b)); r.assertLogContains("script file contents: ", b); - r.assertLogContains("Started container jnlp", b); assertFalse("There are pods leftover after test execution, see previous logs", deletePods(cloud.connect(), getLabels(cloud, this, name), true)); } @@ -288,7 +286,6 @@ public void inheritFrom() throws Exception { standard.setName("standard"); cloud.addTemplate(standard); r.assertBuildStatusSuccess(r.waitForCompletion(b)); - r.assertLogContains("Created container jnlp", b); } @Test