TextCollectingVisitor works better with code blocks (#575)

* Avoid duplicated setLastNode(..) call

The call was happening first irrespective of whether the node was a TextContainer so lifted this logic out of the `if` block.

* Avoid duplicate LineBreakNode logic

It seems that currently LineBreakNode is independant of TextContainer so it seems safe to pull that out of the `if` block.

* Paragraph with trailing blank line collects trailing blank line

Consequently only a single new line is needed to add paragraph breaks

* Collect a blank line before all (except first) Block instances

- Removed special case logic for Paragraph blocks
- Generalised from BlankLineBreakNode case

* IndentedCodeBlock implements TextContainer

Implementation based on Text collectText(..) but using getContentChars()

flexmark-core-test/src/test/java/com/vladsch/flexmark/core/test/util/parser/ast/TextCollectingVisitorTest.java

@@ -67,4 +67,48 @@ public void test_pararaphs() {
                 "with more text" +
                 "", text);
     }
+
+    @Test
+    public void test_paragraph_and_fenced_code_block() {
+        Parser parser = Parser.builder().build();
+        Node document = parser.parse("" +
+                "before\n" +
+                "\n" +
+                "```\n" +
+                "fenced code\n" +
+                "block\n" +
+                "```\n" +
+                "\n" +
+                "after");
+        TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
+        final String text = collectingVisitor.collectAndGetText(document);
+        assertEquals("" +
+                "before\n" +
+                "\n"+
+                "fenced code\n" +
+                "block\n" +
+                "\n"+
+                "after" +
+                "", text);
+    }
+
+    @Test
+    public void test_paragraph_and_indented_code_block() {
+        Parser parser = Parser.builder().build();
+        Node document = parser.parse("" +
+                "before\n" +
+                "\n" +
+                "    indented code block\n" +
+                "\n" +
+                "after");
+        TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
+        final String text = collectingVisitor.collectAndGetText(document);
+        assertEquals("" +
+                "before\n" +
+                "\n"+
+                "indented code block\n" +
+                "\n"+
+                "after" +
+                "", text);
+    }
 }

flexmark-util-ast/src/main/java/com/vladsch/flexmark/util/ast/TextCollectingVisitor.java

@@ -15,26 +15,24 @@ public TextCollectingVisitor() {
             @Override
             public void processNode(@NotNull Node node, boolean withChildren, @NotNull BiConsumer<Node, Visitor<Node>> processor) {
                 if (!node.isOrDescendantOfType(DoNotCollectText.class)) {
+                    out.setLastNode(node);
+                    if (node instanceof Block && out.isNotEmpty()) {
+                        out.appendEol();
+                    }
                     if (node instanceof TextContainer) {
-                        out.setLastNode(node);
-                        if (((TextContainer) node).collectText(out, flags, myVisitor)) {
+                        final TextContainer textContainer = (TextContainer) node;
+                        if (textContainer.collectText(out, flags, myVisitor)) {
                             if (node instanceof BlankLineBreakNode && out.isNotEmpty()) {
                                 out.appendEol();
                             }
                             processChildren(node, processor);
-                            if (node instanceof LineBreakNode && out.needEol()) {
-                                out.appendEol();
-                            }
                         }
+                        textContainer.collectEndText(out, flags, myVisitor);
                     } else {
-                        out.setLastNode(node);
-                        if (node instanceof BlankLineBreakNode && out.isNotEmpty()) {
-                            out.appendEol();
-                        }
                         processChildren(node, processor);
-                        if (node instanceof LineBreakNode && out.needEol()) {
-                            out.appendEol();
-                        }
+                    }
+                    if (node instanceof LineBreakNode && out.needEol()) {
+                        out.appendEol();
                     }
                 }
             }

flexmark-util-ast/src/main/java/com/vladsch/flexmark/util/ast/TextContainer.java

@@ -53,4 +53,14 @@ public int getBits() {
      * @return true if child nodes should be visited
      */
     boolean collectText(ISequenceBuilder<? extends ISequenceBuilder<?, BasedSequence>, BasedSequence> out, int flags, NodeVisitor nodeVisitor);
+
+    /**
+     * Append node's text ending, after any child nodes have been visited.
+     * The default implementation does nothing.
+     *
+     * @param out   sequence build to which to append text
+     * @param flags collection flags
+     * @param nodeVisitor node visitor to use to visit children
+     */
+    default void collectEndText(ISequenceBuilder<? extends ISequenceBuilder<?, BasedSequence>, BasedSequence> out, int flags, NodeVisitor nodeVisitor) {}
 }

flexmark/src/main/java/com/vladsch/flexmark/ast/IndentedCodeBlock.java

@@ -2,12 +2,19 @@
 
 import com.vladsch.flexmark.util.ast.Block;
 import com.vladsch.flexmark.util.ast.BlockContent;
+import com.vladsch.flexmark.util.ast.NodeVisitor;
+import com.vladsch.flexmark.util.ast.TextContainer;
 import com.vladsch.flexmark.util.sequence.BasedSequence;
+import com.vladsch.flexmark.util.sequence.Escaping;
+import com.vladsch.flexmark.util.sequence.ReplacedTextMapper;
+import com.vladsch.flexmark.util.sequence.builder.ISequenceBuilder;
 import org.jetbrains.annotations.NotNull;
 
 import java.util.List;
 
-public class IndentedCodeBlock extends Block {
+import static com.vladsch.flexmark.util.misc.BitFieldSet.any;
+
+public class IndentedCodeBlock extends Block implements TextContainer {
 
     @NotNull
     @Override
@@ -29,4 +36,19 @@ public IndentedCodeBlock(BasedSequence chars, List<BasedSequence> segments) {
     public IndentedCodeBlock(BlockContent blockContent) {
         super(blockContent);
     }
+
+    @Override
+    public boolean collectText(ISequenceBuilder<? extends ISequenceBuilder<?, BasedSequence>, BasedSequence> out, int flags, NodeVisitor nodeVisitor) {
+        final BasedSequence chars = getContentChars();
+        if (any(flags, F_NODE_TEXT)) {
+            out.append(chars);
+        } else {
+            ReplacedTextMapper textMapper = new ReplacedTextMapper(chars);
+            BasedSequence unescaped = Escaping.unescape(chars, textMapper);
+            if (!unescaped.isEmpty()) {
+                out.append(unescaped);
+            }
+        }
+        return false;
+    }
 }

flexmark/src/main/java/com/vladsch/flexmark/ast/Paragraph.java

@@ -135,9 +135,13 @@ public boolean hasTableSeparator() {
 
     @Override
     public boolean collectText(ISequenceBuilder<? extends ISequenceBuilder<?, BasedSequence>, BasedSequence> out, int flags, NodeVisitor nodeVisitor) {
-        if (!out.isEmpty()) {
-            out.add("\n\n");
-        }
         return true;
     }
+
+    @Override
+    public void collectEndText(ISequenceBuilder<? extends ISequenceBuilder<?, BasedSequence>, BasedSequence> out, int flags, NodeVisitor nodeVisitor) {
+        if (trailingBlankLine) {
+            out.add("\n");
+        }
+    }
 }