#185 Add RunListener.onFinalized listener. Inject START ActionNote

.gitattributes

@@ -0,0 +1 @@
+src/test/resources/hudson/plugins/ansicolor/action/ShortlogActionCreatorTest/testlog-crlf.log eol=crlf

CHANGELOG.md

@@ -1,6 +1,8 @@
 0.7.2 (Next)
 ============
 
+* [#197](https://github.com/jenkinsci/ansicolor-plugin/pull/197): Recognize timestamper's GlobalDecorator from extension list  - [@tszmytka](https://github.com/tszmytka).
+* [#185](https://github.com/jenkinsci/ansicolor-plugin/pull/185): Render escape codes correctly in shortlog - [@tszmytka](https://github.com/tszmytka).
 * Your contribution here.
 
 

pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.jenkins-ci.plugins</groupId>
     <artifactId>plugin</artifactId>
-    <version>4.3</version>
+    <version>4.4</version>
     <relativePath />
   </parent>
 
@@ -30,6 +30,11 @@
       <name>Daniel Doubrovkine</name>
       <email>dblock@dblock.org</email>
     </developer>
+    <developer>
+      <id>tszmytka</id>
+      <name>Tomek Szmytka</name>
+      <email>tszmytka@gmail.com</email>
+    </developer>
   </developers>
 
   <scm>

src/main/java/hudson/plugins/ansicolor/ColorConsoleAnnotator.java

@@ -31,6 +31,7 @@
 import hudson.model.Queue;
 import hudson.model.Run;
 import hudson.plugins.ansicolor.action.ColorizedAction;
+import hudson.plugins.ansicolor.action.LineIdentifier;
 import jenkins.model.Jenkins;
 import org.apache.commons.io.output.CountingOutputStream;
 import org.apache.commons.io.output.NullOutputStream;
@@ -59,25 +60,38 @@ final class ColorConsoleAnnotator extends ConsoleAnnotator<Object> {
 
     private final String defaultColorMapName;
 
+    private final LineIdentifier lineIdentifier;
+
     @CheckForNull
     private String colorMapName;
 
     @Nonnull
     private List<AnsiAttributeElement> openTags = Collections.emptyList();
 
-    private ColorConsoleAnnotator(String defaultColorMapName) {
+    private long lineNo;
+
+    private ColorConsoleAnnotator(String defaultColorMapName, LineIdentifier lineIdentifier, long startLineNo) {
         this.defaultColorMapName = defaultColorMapName;
+        this.lineIdentifier = lineIdentifier;
+        this.lineNo = startLineNo;
     }
 
     @Override
     public ConsoleAnnotator<Object> annotate(@Nonnull Object context, @Nonnull MarkupText text) {
-        final ColorizedAction colorizedAction = ColorizedAction.parseAction(text, runOf(context));
+        lineNo++;
+        Run<?, ?> run = runOf(context);
+        if (run == null) {
+            return this;
+        }
+        final ColorizedAction colorizedAction = lineNo == 1
+            ? ColorizedAction.parseAction(text.getText(), lineNo, run, lineIdentifier)
+            : ColorizedAction.parseAction(text, run);
         switch (colorizedAction.getCommand()) {
             case START:
                 colorMapName = colorizedAction.getColorMapName();
                 break;
             case STOP:
-                return FACTORY.newInstance(context);
+                return FACTORY.newInstance(context, lineNo);
             case IGNORE:
                 return this;
             default:
@@ -212,7 +226,11 @@ public static final class Factory extends ConsoleAnnotatorFactory<Object> {
 
         @Override
         public ConsoleAnnotator<Object> newInstance(Object context) {
-            return new ColorConsoleAnnotator(Jenkins.get().getDescriptorByType(AnsiColorBuildWrapper.DescriptorImpl.class).getGlobalColorMapName());
+            return newInstance(context, 0);
+        }
+
+        private ConsoleAnnotator<Object> newInstance(Object context, long startLineNo) {
+            return new ColorConsoleAnnotator(Jenkins.get().getDescriptorByType(AnsiColorBuildWrapper.DescriptorImpl.class).getGlobalColorMapName(), new LineIdentifier(), startLineNo);
         }
     }
 }

src/main/java/hudson/plugins/ansicolor/action/ActionNote.java

@@ -16,7 +16,7 @@ public class ActionNote extends ConsoleNote<Run<?, ?>> {
     private final String actionId;
 
     public ActionNote(ColorizedAction action) {
-        actionId = action.getId().toString();
+        actionId = action.getId();
     }
 
     @Override

src/main/java/hudson/plugins/ansicolor/action/ColorizedAction.java

@@ -41,7 +41,7 @@ public class ColorizedAction extends InvisibleAction {
     static final ColorizedAction CONTINUE = new ColorizedAction("", Command.CONTINUE);
     static final ColorizedAction IGNORE = new ColorizedAction("", Command.IGNORE);
 
-    private final UUID id;
+    private final String id;
 
     private final String colorMapName;
 
@@ -55,12 +55,18 @@ public enum Command {
     }
 
     public ColorizedAction(String colorMapName, Command command) {
-        id = UUID.randomUUID();
+        id = UUID.randomUUID().toString();
         this.colorMapName = colorMapName == null || colorMapName.isEmpty() ? AnsiColorMap.DefaultName : colorMapName;
         this.command = command;
     }
 
-    public UUID getId() {
+    public ColorizedAction(String id, ColorizedAction other) {
+        this.id = id;
+        colorMapName = other.colorMapName;
+        command = other.command;
+    }
+
+    public String getId() {
         return id;
     }
 
@@ -79,8 +85,12 @@ public static ColorizedAction parseAction(MarkupText text, Run<?, ?> run) {
             final int from = actionIdOffset + TAG_ACTION_BEGIN.length() + 1;
             final int to = line.indexOf("\"", from);
             final String id = line.substring(from, to);
-            return run.getActions(ColorizedAction.class).stream().filter(a -> id.equals(a.getId().toString())).findAny().orElse(CONTINUE);
+            return run.getActions(ColorizedAction.class).stream().filter(a -> id.equals(a.getId())).findAny().orElse(CONTINUE);
         }
         return line.contains(TAG_PIPELINE_INTERNAL) ? IGNORE : CONTINUE;
     }
+
+    public static ColorizedAction parseAction(String lineContent, long lineNo, Run<?, ?> run, LineIdentifier lineIdentifier) {
+        return run.getActions(ColorizedAction.class).stream().filter(a -> lineIdentifier.isEqual(lineContent, lineNo, a.id)).findAny().orElse(CONTINUE);
+    }
 }

src/main/java/hudson/plugins/ansicolor/action/LineIdentifier.java

@@ -0,0 +1,34 @@
+package hudson.plugins.ansicolor.action;
+
+import java.io.Serializable;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+public class LineIdentifier implements Serializable {
+    private static final String ALGORITHM = "SHA-256";
+    private static final long serialVersionUID = 1;
+    private transient MessageDigest messageDigest;
+
+    private MessageDigest getMessageDigest() {
+        if (messageDigest == null) {
+            try {
+                messageDigest = MessageDigest.getInstance(ALGORITHM);
+            } catch (NoSuchAlgorithmException e) {
+                throw new IllegalStateException("Cannot get message digest", e);
+            }
+        }
+        return messageDigest;
+    }
+
+    public String hash(String lineContent, long lineNo) {
+        final String key = String.join("|", lineContent, String.valueOf(lineNo));
+        return Base64.getEncoder().encodeToString(getMessageDigest().digest(key.getBytes(UTF_8)));
+    }
+
+    public boolean isEqual(String lineContent, long lineNo, String other) {
+        return hash(lineContent, lineNo).equals(other);
+    }
+}

src/main/java/hudson/plugins/ansicolor/action/ShortlogActionCreator.java

@@ -0,0 +1,146 @@
+package hudson.plugins.ansicolor.action;
+
+import hudson.Extension;
+import hudson.console.ConsoleNote;
+import hudson.model.Run;
+import hudson.model.listeners.RunListener;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+public class ShortlogActionCreator {
+    private static final Logger LOGGER = Logger.getLogger(ShortlogActionCreator.class.getName());
+    private static final int CONSOLE_TAIL_DEFAULT = 150;
+    private static final int BUFFER_SIZE = 16 * 1024;
+
+    private final LineIdentifier lineIdentifier;
+    private final byte[] eol;
+
+    public ShortlogActionCreator(LineIdentifier lineIdentifier, String eol) {
+        this.lineIdentifier = lineIdentifier;
+        this.eol = eol.getBytes(UTF_8);
+    }
+
+    public ColorizedAction createActionForShortlog(File logFile, Map<String, ColorizedAction> actions, int shortlogLimit) {
+        final ActionContext lastAction = findLastActionBefore(logFile, actions.keySet(), shortlogLimit);
+        if (!lastAction.isEmpty()) {
+            final ColorizedAction colorizedAction = actions.get(lastAction.serializedAction);
+            if (ColorizedAction.Command.START.equals(colorizedAction.getCommand())) {
+                return new ColorizedAction(lineIdentifier.hash(ConsoleNote.removeNotes(lastAction.line), 1), colorizedAction);
+            }
+        }
+        return null;
+    }
+
+    private ActionContext findLastActionBefore(File logFile, Collection<String> serializedActions, int shortlogLimit) {
+        try (BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(logFile))) {
+            final long shortlogStart = logFile.length() - shortlogLimit * 1024L;
+            if (shortlogStart > 0) {
+                final byte[] buf = new byte[BUFFER_SIZE];
+                int read;
+                int totalRead = 0;
+                String lastAction = "";
+                String partialLine = "";
+                while ((read = inputStream.read(buf)) != -1) {
+                    final int startInBuff = shortlogStart > totalRead ? (int) (shortlogStart - totalRead) : 0;
+                    final String action = findLastAction(serializedActions, buf, startInBuff);
+                    if (!action.isEmpty()) {
+                        lastAction = action;
+                    }
+                    if (totalRead + read >= shortlogStart) {
+                        final int eolPos = indexOfEol(buf, startInBuff);
+                        if (eolPos != -1 && !lastAction.isEmpty()) {
+                            return new ActionContext(lastAction, partialLine + new String(buf, startInBuff, eolPos - startInBuff + eol.length, UTF_8));
+                        } else {
+                            // line extends to the next buffer
+                            partialLine = new String(Arrays.copyOfRange(buf, startInBuff, buf.length), UTF_8);
+                        }
+                    }
+                    totalRead += read;
+                }
+            }
+        } catch (IOException e) {
+            LOGGER.warning("Cannot search log for actions: " + e.getMessage());
+        }
+        return new ActionContext();
+    }
+
+    private String findLastAction(Collection<String> serializedActions, byte[] buf, int maxPos) {
+        String lastAction = "";
+        int preamblePos = 0;
+        while (preamblePos < maxPos && (preamblePos = ConsoleNote.findPreamble(buf, preamblePos, buf.length - preamblePos)) != -1) {
+            final int begin = preamblePos;
+            lastAction = serializedActions.stream().filter(sa -> buf.length - begin > sa.length() && sa.equals(new String(buf, begin, sa.length(), UTF_8))).findFirst().orElse(lastAction);
+            preamblePos++;
+        }
+        return lastAction;
+    }
+
+    private int indexOfEol(byte[] buf, int after) {
+        for (int i = after; i < buf.length; i++) {
+            if (Arrays.equals(Arrays.copyOfRange(buf, i, i + eol.length), eol)) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    @Extension
+    public static class Listener extends RunListener<Run<?, ?>> {
+        @Override
+        public void onFinalized(Run<?, ?> run) {
+            super.onFinalized(run);
+            final List<ColorizedAction.Command> commands = Arrays.asList(ColorizedAction.Command.START, ColorizedAction.Command.STOP);
+            Map<String, ColorizedAction> actions = run.getActions(ColorizedAction.class).stream()
+                .filter(a -> commands.contains(a.getCommand()))
+                .collect(Collectors.toMap(a -> {
+                    try {
+                        return new ActionNote(a).encode();
+                    } catch (IOException e) {
+                        LOGGER.warning("Will not be able to identify all ColorizedActions: " + e.getMessage());
+                    }
+                    return "";
+                }, Function.identity()));
+            if (!actions.isEmpty()) {
+                final File logFile = new File(run.getRootDir(), "log");
+                if (logFile.isFile()) {
+                    final ShortlogActionCreator shortlogActionCreator = new ShortlogActionCreator(new LineIdentifier(), System.lineSeparator());
+                    final String consoleTail = System.getProperty("hudson.consoleTailKB");
+                    final ColorizedAction action = shortlogActionCreator.createActionForShortlog(logFile, actions, consoleTail != null ? Integer.parseInt(consoleTail) : CONSOLE_TAIL_DEFAULT);
+                    if (action != null) {
+                        run.addAction(action);
+                    }
+                }
+            }
+        }
+    }
+
+    private static class ActionContext {
+        private final String serializedAction;
+        private final String line;
+
+        public ActionContext() {
+            this(null, null);
+        }
+
+        public ActionContext(String serializedAction, String line) {
+            this.serializedAction = serializedAction;
+            this.line = line;
+        }
+
+        public boolean isEmpty() {
+            return serializedAction == null && line == null;
+        }
+    }
+}

src/test/java/hudson/plugins/ansicolor/JenkinsTestSupport.java

@@ -0,0 +1,51 @@
+package hudson.plugins.ansicolor;
+
+import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition;
+import org.jenkinsci.plugins.workflow.job.WorkflowJob;
+import org.jenkinsci.plugins.workflow.job.WorkflowRun;
+import org.junit.Rule;
+import org.junit.runners.model.Statement;
+import org.jvnet.hudson.test.RestartableJenkinsRule;
+
+import java.io.File;
+import java.io.StringWriter;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class JenkinsTestSupport {
+    @Rule
+    public RestartableJenkinsRule jenkinsRule = new RestartableJenkinsRule();
+    private static final int CONSOLE_TAIL_DEFAULT = 150;
+
+    protected void assertOutputOnRunningPipeline(String expectedOutput, String notExpectedOutput, String pipelineScript, boolean useShortLog) {
+        assertOutputOnRunningPipeline(Collections.singletonList(expectedOutput), Collections.singletonList(notExpectedOutput), pipelineScript, useShortLog);
+    }
+
+    protected void assertOutputOnRunningPipeline(Collection<String> expectedOutput, Collection<String> notExpectedOutput, String pipelineScript, boolean useShortLog) {
+        jenkinsRule.addStep(new Statement() {
+
+            @Override
+            public void evaluate() throws Throwable {
+                final WorkflowJob project = jenkinsRule.j.jenkins.createProject(WorkflowJob.class, "p");
+                project.setDefinition(new CpsFlowDefinition(pipelineScript, true));
+                jenkinsRule.j.assertBuildStatusSuccess(project.scheduleBuild2(0));
+                StringWriter writer = new StringWriter();
+                final WorkflowRun lastBuild = project.getLastBuild();
+                final long start = useShortLog ? new File(lastBuild.getRootDir(), "log").length() - CONSOLE_TAIL_DEFAULT * 1024 : 0;
+                assertTrue(lastBuild.getLogText().writeHtmlTo(start, writer) > 0);
+                final String html = writer.toString().replaceAll("<!--.+?-->", "");
+                assertThat(html).contains(expectedOutput);
+                assertThat(html).doesNotContain(notExpectedOutput);
+            }
+        });
+    }
+
+    protected static String repeat(String s, int times) {
+        return IntStream.range(0, times).mapToObj(i -> s).collect(Collectors.joining());
+    }
+}

src/test/java/hudson/plugins/ansicolor/action/ActionNoteTest.java

@@ -30,7 +30,7 @@ public class ActionNoteTest {
 
     @Before
     public void setUp() throws Exception {
-        when(colorizedAction.getId()).thenReturn(UUID);
+        when(colorizedAction.getId()).thenReturn(UUID.toString());
         actionNote = new ActionNote(colorizedAction);
     }