[CS2] Fix handling of tabbed code blocks in .litcoffee files (#4485)

* Add tabbed literate test; modernize Markdown title heading

* Better parsing of Literate CoffeeScript files, including now correct parsing of tabbed .litcoffee files; and more accurate stack traces (assuming you don’t do your own word wrapping within list items)

* Swap Marked for MarkdownIt for parsing the Markdown of Literate CoffeeScript files; use MarkdownIt’s `map` property to preserve correct line numbers

* Literate CoffeeScript tests: remove trailing whitespace, fix spelling

* Literate CoffeeScript tests: add block quote test

* Literate CoffeeScript (tabbed, at least) seems to need a consistent starting indentation

* Restore test

* Reference links work now in MarkdownIt

* Breaking change in Literate CoffeeScript: code blocks within HTML tags must be unindented

* Breaking change in Literate CoffeeScript: code blocks within lists need a blank line separating them from the list item text

package.json

@@ -47,6 +47,6 @@
     "underscore": "~1.8.3"
   },
   "dependencies": {
-    "marked": "~0.3.6"
+    "markdown-it": "^8.3.1"
   }
 }

src/helpers.coffee

@@ -2,16 +2,7 @@
 # the **Lexer**, **Rewriter**, and the **Nodes**. Merge objects, flatten
 # arrays, count characters, that sort of thing.
 
-marked = require 'marked'
-# marked.setOptions
-#   renderer: new marked.Renderer()
-#   gfm: true
-#   tables: true
-#   breaks: false
-#   pedantic: false
-#   sanitize: true
-#   smartLists: true
-#   smartypants: false
+md = require('markdown-it')()
 
 # Peek at the beginning of a given string to see if it matches a sequence.
 exports.starts = (string, literal, start) ->
@@ -80,23 +71,18 @@ exports.some = Array::some ? (fn) ->
 
 # Simple function for extracting code from Literate CoffeeScript by stripping
 # out all non-code blocks, producing a string of CoffeeScript code that can
-# be compiled “normally.”
+# be compiled “normally.” Uses [MarkdownIt](https://markdown-it.github.io/)
+# to tell the difference between Markdown and code blocks.
 exports.invertLiterate = (code) ->
-  # Create a placeholder for tabs, that isn’t used anywhere in `code`, and then
-  # re-insert the tabs after code extraction.
-  generateRandomToken = ->
-    "#{Math.random() * Date.now()}"
-  while token is undefined or code.indexOf(token) isnt -1
-    token = generateRandomToken()
-
-  code = code.replace "\t", token
-  # Parse as markdown, discard everything except code blocks.
-  out = ""
-  for item in marked.lexer code, {}
-    out += "#{item.text}\n" if item.type is 'code'
-  # Put the tabs back in.
-  out.replace token, "\t"
-  out
+  out = []
+  md.renderer.rules =
+    code_block: (tokens, idx) ->
+      startLine = tokens[idx].map[0]
+      lines = tokens[idx].content.split '\n'
+      for line, i in lines
+        out[startLine + i] = line
+  md.render code
+  out.join '\n'
 
 # Merge two jison-style location data objects together.
 # If `last` is not provided, this will simply return `first`.

test/literate.litcoffee

@@ -1,19 +1,18 @@
-Literate CoffeeScript Test
---------------------------
+# Literate CoffeeScript Test
 
 comment comment
 
     test "basic literate CoffeeScript parsing", ->
       ok yes
-      
+
 now with a...
-  
+
     test "broken up indentation", ->
-    
+
 ... broken up ...
 
       do ->
-      
+
 ... nested block.
 
         ok yes
@@ -25,36 +24,36 @@ Code must be separated from text by a blank line.
 The next line is part of the text and will not be executed.
       fail()
 
-      ok yes        
-        
+      ok yes
+
 Code in `backticks is not parsed` and...
 
     test "comments in indented blocks work", ->
       do ->
         do ->
           # Regular comment.
-          
+
           ###
             Block comment.
           ###
-          
+
           ok yes
-          
-Regular [Markdown](http://example.com/markdown) features, like links 
+
+Regular [Markdown](http://example.com/markdown) features, like links
 and unordered lists, are fine:
 
-  * I 
-  
+  * I
+
   * Am
-  
+
   * A
-  
+
   * List
 
 Tabs work too:
 
-				test "tabbed code", ->
-					ok yes
+	test "tabbed code", ->
+		ok yes
 
 ---
 
@@ -63,11 +62,12 @@ Tabs work too:
 
 <p>
 
-    executed = true # should not execute, this is just HTML para, not code!
+if true
+  executed = true # should not execute, this is just HTML para, not code!
 
 </p>
 
-    test "should ignore indented sections inside HTML", ->
+    test "should ignore code blocks inside HTML", ->
       eq executed, false
 
 ---
@@ -119,24 +119,8 @@ Tabs work too:
 
 ---
 
-This next one probably passes because a string is inoffensive in compiled js, also, can't get `marked` to parse it correctly, and not sure if empty line is permitted between title and reference
-
 This is [an example][id] reference-style link.
-[id]: http://example.com/
-
-    "Optional Title Here"
-
----
-
-    executed = no
-
-1986. What a great season.
-          executed = yes
-
-and test...
-
-    test "should recognise indented code blocks in lists", ->
-      ok executed
+[id]: http://example.com/  "Optional Title Here"
 
 ---
 
@@ -148,7 +132,7 @@ and test...
 
 and test...
 
-    test "should recognise indented code blocks in lists with empty line as separator", ->
+    test "should recognize indented code blocks in lists with empty line as separator", ->
       ok executed
 
 ---
@@ -163,3 +147,11 @@ and test...
     test "should ignore indented code in escaped list like number", ->
       eq executed, no
 
+one last test!
+
+    test "block quotes should render correctly", ->
+      quote = '''
+        foo
+           and bar!
+      '''
+      eq quote, 'foo\n   and bar!'