From 5c2ae3aa025265c9d6f92389d281214ea0bd515d Mon Sep 17 00:00:00 2001 From: M Pacer Date: Fri, 23 Jun 2017 11:51:01 -0700 Subject: [PATCH 1/5] Improve, normalise mathjax handling, yapf the file --- nbconvert/filters/markdown_mistune.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/nbconvert/filters/markdown_mistune.py b/nbconvert/filters/markdown_mistune.py index eb0f119cd..445ceba18 100644 --- a/nbconvert/filters/markdown_mistune.py +++ b/nbconvert/filters/markdown_mistune.py @@ -19,16 +19,19 @@ from pygments.util import ClassNotFound from nbconvert.filters.strings import add_anchor -from nbconvert.utils.exceptions import ConversionException + +block_math = re.compile(r"^\$\$(.*?)\$\$|^\\\\\[(.*?)\\\\\]", re.DOTALL) class MathBlockGrammar(mistune.BlockGrammar): - block_math = re.compile(r"^\$\$(.*?)\$\$", re.DOTALL) + block_math = block_math latex_environment = re.compile(r"^\\begin\{([a-z]*\*?)\}(.*?)\\end\{\1\}", - re.DOTALL) + re.DOTALL) + class MathBlockLexer(mistune.BlockLexer): - default_rules = ['block_math', 'latex_environment'] + mistune.BlockLexer.default_rules + default_rules = ['block_math', 'latex_environment' + ] + mistune.BlockLexer.default_rules def __init__(self, rules=None, **kwargs): if rules is None: @@ -39,7 +42,7 @@ def parse_block_math(self, m): """Parse a $$math$$ block""" self.tokens.append({ 'type': 'block_math', - 'text': m.group(1) + 'text': m.group(1) or m.group(2) }) def parse_latex_environment(self, m): @@ -52,7 +55,7 @@ def parse_latex_environment(self, m): class MathInlineGrammar(mistune.InlineGrammar): math = re.compile(r"^\$(.+?)\$", re.DOTALL) - block_math = re.compile(r"^\$\$(.+?)\$\$", re.DOTALL) + block_math = block_math text = re.compile(r'^[\s\S]+?(?=[\\ Date: Fri, 23 Jun 2017 12:39:11 -0700 Subject: [PATCH 2/5] add inline correction, use multiple groups everywhere, change property and function name --- nbconvert/filters/markdown_mistune.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nbconvert/filters/markdown_mistune.py b/nbconvert/filters/markdown_mistune.py index 445ceba18..035bcc58a 100644 --- a/nbconvert/filters/markdown_mistune.py +++ b/nbconvert/filters/markdown_mistune.py @@ -21,6 +21,7 @@ from nbconvert.filters.strings import add_anchor block_math = re.compile(r"^\$\$(.*?)\$\$|^\\\\\[(.*?)\\\\\]", re.DOTALL) +inline_math = re.compile(r"^\$(.+?)\$|^\\\\\((.+?)\\\\\)", re.DOTALL) class MathBlockGrammar(mistune.BlockGrammar): @@ -54,24 +55,25 @@ def parse_latex_environment(self, m): class MathInlineGrammar(mistune.InlineGrammar): - math = re.compile(r"^\$(.+?)\$", re.DOTALL) + inline_math = inline_math block_math = block_math text = re.compile(r'^[\s\S]+?(?=[\\ Date: Fri, 23 Jun 2017 13:09:12 -0700 Subject: [PATCH 3/5] make tests check for this, but now tests fail --- nbconvert/filters/tests/test_markdown.py | 29 ++++++++++++++++-------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/nbconvert/filters/tests/test_markdown.py b/nbconvert/filters/tests/test_markdown.py index bec80e070..7ba023277 100644 --- a/nbconvert/filters/tests/test_markdown.py +++ b/nbconvert/filters/tests/test_markdown.py @@ -127,15 +127,26 @@ def test_markdown2html_math(self): ("$$\n" "a = 1 *3* 5\n" "$$"), - "$ a = 1 *3* 5 $", - "$s_i = s_{i}\n$", - "$aa;a-b<0$", - "$$", - "$$aa;a-b<0$$", - "$$$$", - """$ + "$ a = 1 *3* 5 $", + "$s_i = s_{i}\n$", + "$aa;a-b<0$", + "$$", + "$$aa;a-b<0$$", + "$$$$", + ("\\\\[\n" + "a = 1 *3* 5\n" + "\\\\]"), + "\\\\( a = 1 *3* 5 \\\\)", + "\\\\(s_i = s_{i}\n\\\\)", + "\\\\(aa;a-b<0\\\\)", + "\\\\(\\\\)", + "\\\\[aa;a-b<0\\\\]", + "\\\\[\\\\]", + """$ \\begin{tabular}{ l c r } 1 & 2 & 3 \\ 4 & 5 & 6 \\ From 593ce624b98c3937a9aa3417252fe52aa79855c8 Mon Sep 17 00:00:00 2001 From: M Pacer Date: Fri, 23 Jun 2017 13:38:43 -0700 Subject: [PATCH 4/5] partial yapf, remove \\( syntax from unchanged test --- nbconvert/filters/tests/test_markdown.py | 89 +++++++++++------------- 1 file changed, 42 insertions(+), 47 deletions(-) diff --git a/nbconvert/filters/tests/test_markdown.py b/nbconvert/filters/tests/test_markdown.py index 7ba023277..8e51241d0 100644 --- a/nbconvert/filters/tests/test_markdown.py +++ b/nbconvert/filters/tests/test_markdown.py @@ -52,9 +52,10 @@ class TestMarkdown(TestsBase): def test_markdown2latex(self): """markdown2latex test""" for index, test in enumerate(self.tests): - self._try_markdown(partial(convert_pandoc, from_format='markdown', - to_format='latex'), test, - self.tokens[index]) + self._try_markdown( + partial( + convert_pandoc, from_format='markdown', to_format='latex'), + test, self.tokens[index]) @dec.onlyif_cmds_exist('pandoc') def test_markdown2latex_markup(self): @@ -108,45 +109,35 @@ def test_markdown2html(self): self._try_markdown(markdown2html, test, self.tokens[index]) def test_markdown2html_heading_anchors(self): - for md, tokens in [ - ('# test', - ('test', 'id="test"', u'¶', "anchor-link") - ), - ('###test head space', - ('test head space', 'id="test-head-space"', u'¶', "anchor-link") - ) - ]: + for md, tokens in [('# test', ('test', 'id="test"', + u'¶', "anchor-link")), + ('###test head space', + ('test head space', 'id="test-head-space"', + u'¶', "anchor-link"))]: self._try_markdown(markdown2html, md, tokens) def test_markdown2html_math(self): - # Mathematical expressions not containing <, >, & should be passed through unaltered + # Mathematical expressions not containing <, >, & + # should be passed through unaltered # all the "<", ">", "&" must be escaped correctly - cases = [("\\begin{equation*}\n" - "\\left( \\sum_{k=1}^n a_k b_k \\right)^2 \\leq \\left( \\sum_{k=1}^n a_k^2 \\right) \\left( \\sum_{k=1}^n b_k^2 \\right)\n" - "\\end{equation*}"), - ("$$\n" - "a = 1 *3* 5\n" - "$$"), - "$ a = 1 *3* 5 $", - "$s_i = s_{i}\n$", - "$aa;a-b<0$", - "$$", - "$$aa;a-b<0$$", - "$$$$", - ("\\\\[\n" - "a = 1 *3* 5\n" - "\\\\]"), - "\\\\( a = 1 *3* 5 \\\\)", - "\\\\(s_i = s_{i}\n\\\\)", - "\\\\(aa;a-b<0\\\\)", - "\\\\(\\\\)", - "\\\\[aa;a-b<0\\\\]", - "\\\\[\\\\]", - """$ + cases = [( + "\\begin{equation*}\n" + + ("\\left( \\sum_{k=1}^n a_k b_k \\right)^2 " + "\\leq \\left( \\sum_{k=1}^n a_k^2 \\right) " + "\\left( \\sum_{k=1}^n b_k^2 \\right)\n") + + "\\end{equation*}"), + ("$$\n" + "a = 1 *3* 5\n" + "$$"), + "$ a = 1 *3* 5 $", + "$s_i = s_{i}\n$", + "$aa;a-b<0$", + "$$", + "$$aa;a-b<0$$", + "$$$$", + """$ \\begin{tabular}{ l c r } 1 & 2 & 3 \\ 4 & 5 & 6 \\ @@ -156,9 +147,11 @@ def test_markdown2html_math(self): for case in cases: result = markdown2html(case) # find the equation in the generated texts - search_result = re.search("\$.*\$",result,re.DOTALL) + search_result = re.search("\$.*\$", result, re.DOTALL) if search_result is None: - search_result = re.search("\\\\begin\\{equation.*\\}.*\\\\end\\{equation.*\\}",result,re.DOTALL) + search_result = re.search( + "\\\\begin\\{equation.*\\}.*\\\\end\\{equation.*\\}", + result, re.DOTALL) math = search_result.group(0) # the resulting math part can not contain "<", ">" or # "&" not followed by "lt;", "gt;", or "amp;". @@ -167,13 +160,13 @@ def test_markdown2html_math(self): # python 2.7 has assertNotRegexpMatches instead of assertNotRegex if not hasattr(self, 'assertNotRegex'): self.assertNotRegex = self.assertNotRegexpMatches - self.assertNotRegex(math,"&(?![gt;|lt;|amp;])") + self.assertNotRegex(math, "&(?![gt;|lt;|amp;])") # the result should be able to be unescaped correctly - self.assertEquals(case,self._unescape(math)) + self.assertEquals(case, self._unescape(math)) def test_markdown2html_math_mixed(self): """ensure markdown between inline and inline-block math""" - case = """The entries of $C$ are given by the exact formula: + case = """The entries of \\\\(C\\\\) are given by the exact formula: $$ C_{ik} = \sum_{j=1}^n A_{ij} B_{jk} $$ @@ -211,7 +204,7 @@ def test_markdown2html_math_paragraph(self): for case in cases: s = markdown2html(case) - self.assertIn(case,self._unescape(s)) + self.assertIn(case, self._unescape(s)) @dec.onlyif_cmds_exist('pandoc') def test_markdown2rst(self): @@ -223,8 +216,10 @@ def test_markdown2rst(self): tokens[1] = r'\*\*test' for index, test in enumerate(self.tests): - self._try_markdown(partial(convert_pandoc, from_format='markdown', - to_format='rst'), test, tokens[index]) + self._try_markdown( + partial( + convert_pandoc, from_format='markdown', to_format='rst'), + test, tokens[index]) def _try_markdown(self, method, test, tokens): results = method(test) @@ -234,7 +229,7 @@ def _try_markdown(self, method, test, tokens): for token in tokens: self.assertIn(token, results) - def _unescape(self,s): + def _unescape(self, s): # undo cgi.escape() manually # We must be careful here for compatibility # html.unescape() is not availale on python 2.7 From 83cb6938ade5015326b0c831a876faccf4503655 Mon Sep 17 00:00:00 2001 From: M Pacer Date: Fri, 23 Jun 2017 14:02:50 -0700 Subject: [PATCH 5/5] now the filter tests this and it works --- nbconvert/filters/tests/test_markdown.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nbconvert/filters/tests/test_markdown.py b/nbconvert/filters/tests/test_markdown.py index 8e51241d0..943f9b894 100644 --- a/nbconvert/filters/tests/test_markdown.py +++ b/nbconvert/filters/tests/test_markdown.py @@ -165,14 +165,20 @@ def test_markdown2html_math(self): self.assertEquals(case, self._unescape(math)) def test_markdown2html_math_mixed(self): - """ensure markdown between inline and inline-block math""" + """ensure markdown between inline and inline-block math works and + test multiple LaTeX markup syntaxes. + """ case = """The entries of \\\\(C\\\\) are given by the exact formula: $$ -C_{ik} = \sum_{j=1}^n A_{ij} B_{jk} +C_{ik} = \sum_{j=1}^n A_{ij} B_{jk}, $$ -but there are many ways to _implement_ this computation. $\approx 2mnp$ flops""" - self._try_markdown(markdown2html, case, - case.replace("_implement_", "implement")) +but you can _implement_ this computation in many ways. +$\approx 2mnp$ flops are needed for \\\\[ C_{ik} = \sum_{j=1}^n A_{ij} B_{jk} \\\\].""" + output_check = (case.replace("_implement_", "implement") + .replace("\\\\(", "$").replace("\\\\)", "$") + .replace("\\\\[", "$$").replace("\\\\]", "$$")) + # these replacements are needed because we use $ and $$ in our html output + self._try_markdown(markdown2html, case, output_check) def test_markdown2html_math_paragraph(self): """these should all parse without modification"""