diff --git a/compiler/docgen.nim b/compiler/docgen.nim index fd5c8b47acec0..37831c72c28f6 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -25,6 +25,7 @@ from std/private/globs import nativeToUnixPath const exportSection = skField docCmdSkip = "skip" + DocColOffset = "## ".len # assuming that a space was added after ## type TSections = array[TSymKind, Rope] @@ -322,8 +323,12 @@ proc genComment(d: PDoc, n: PNode): string = when false: # RFC: to preseve newlines in comments, this would work: comment = comment.replace("\n", "\n\n") - renderRstToOut(d[], parseRst(comment, toFullPath(d.conf, n.info), toLinenumber(n.info), - toColumn(n.info), (var dummy: bool; dummy), d.options, d.conf), result) + renderRstToOut(d[], + parseRst(comment, toFullPath(d.conf, n.info), + toLinenumber(n.info), + toColumn(n.info) + DocColOffset, + (var dummy: bool; dummy), d.options, d.conf), + result) proc genRecCommentAux(d: PDoc, n: PNode): Rope = if n == nil: return nil diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index ea0c079daa93d..585c2ede52497 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -146,7 +146,8 @@ ## .. _Sphinx directives: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html import - os, strutils, rstast, algorithm, lists, sequtils + os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, + std/private/miscdollars type RstParseOption* = enum ## options for the RST parser @@ -477,9 +478,10 @@ type EParseError* = object of ValueError const - LineRstInit* = 1 ## Initial line number for standalone RST text - ColRstInit* = 0 ## Initial column number for standalone RST text - ## (Nim global reporting adds ColOffset=1) + LineRstInit* = 1 ## Initial line number for standalone RST text + ColRstInit* = 0 ## Initial column number for standalone RST text + ## (Nim global reporting adds ColOffset=1) + ColRstOffset* = 1 ## 1: a replica of ColOffset for internal use template currentTok(p: RstParser): Token = p.tok[p.idx] template prevTok(p: RstParser): Token = p.tok[p.idx - 1] @@ -487,7 +489,7 @@ template nextTok(p: RstParser): Token = p.tok[p.idx + 1] proc whichMsgClass*(k: MsgKind): MsgClass = ## returns which message class `k` belongs to. - case ($k)[1] + case k.symbolName[1] of 'e', 'E': result = mcError of 'w', 'W': result = mcWarning of 'h', 'H': result = mcHint @@ -497,7 +499,9 @@ proc defaultMsgHandler*(filename: string, line, col: int, msgkind: MsgKind, arg: string) = let mc = msgkind.whichMsgClass let a = $msgkind % arg - let message = "$1($2, $3) $4: $5" % [filename, $line, $col, $mc, a] + var message: string + toLocation(message, filename, line, col + ColRstOffset) + message.add " $1: $2" % [$mc, a] if mc == mcError: raise newException(EParseError, message) else: writeLine(stdout, message) @@ -1954,25 +1958,32 @@ proc parseEnumList(p: var RstParser): PRstNode = if match(p, p.idx, wildcards[w]): break inc w assert w < wildcards.len + proc checkAfterNewline(p: RstParser, report: bool): bool = + ## If no indentation on the next line then parse as a normal paragraph + ## according to the RST spec. And report a warning with suggestions let j = tokenAfterNewline(p, start=p.idx+1) + let requiredIndent = p.tok[p.idx+wildToken[w]].col if p.tok[j].kind notin {tkIndent, tkEof} and - p.tok[j].col < p.tok[p.idx+wildToken[w]].col and + p.tok[j].col < requiredIndent and (p.tok[j].col > col or (p.tok[j].col == col and not match(p, j, wildcards[w]))): if report: let n = p.line + p.tok[j].line let msg = "\n" & """ not enough indentation on line $2 - (if it's continuation of enumeration list), + (should be at column $3 if it's a continuation of enum. list), or no blank line after line $1 (if it should be the next paragraph), or no escaping \ at the beginning of line $1 (if lines $1..$2 are a normal paragraph, not enum. list)""". unindent(8) - rstMessage(p, mwRstStyle, msg % [$(n-1), $n]) + let c = p.col + requiredIndent + ColRstOffset + rstMessage(p, mwRstStyle, msg % [$(n-1), $n, $c], + p.tok[j].line, p.tok[j].col) result = false else: result = true + if not checkAfterNewline(p, report = true): return nil result = newRstNodeA(p, rnEnumList) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index f72ff9e8f0ba9..c52a0fdccd144 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1476,7 +1476,8 @@ $content # ---------- forum --------------------------------------------------------- proc rstToHtml*(s: string, options: RstParseOptions, - config: StringTableRef): string = + config: StringTableRef, + msgHandler: MsgHandler = rst.defaultMsgHandler): string = ## Converts an input rst string into embeddable HTML. ## ## This convenience proc parses any input string using rst markup (it doesn't @@ -1503,11 +1504,10 @@ proc rstToHtml*(s: string, options: RstParseOptions, const filen = "input" var d: RstGenerator - initRstGenerator(d, outHtml, config, filen, options, myFindFile, - rst.defaultMsgHandler) + initRstGenerator(d, outHtml, config, filen, options, myFindFile, msgHandler) var dummyHasToc = false var rst = rstParse(s, filen, line=LineRstInit, column=ColRstInit, - dummyHasToc, options) + dummyHasToc, options, myFindFile, msgHandler) result = "" renderRstToOut(d, rst, result) diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 7d5c0e28f2c27..37becf090caba 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -7,9 +7,34 @@ outputsub: "" import ../../lib/packages/docutils/rstgen import ../../lib/packages/docutils/rst import unittest, strutils, strtabs - -proc toHtml(input: string): string = - rstToHtml(input, {roSupportMarkdown}, defaultConfig()) +import std/private/miscdollars + +proc toHtml(input: string, + rstOptions: RstParseOptions = {roSupportMarkdown}, + error: ref string = nil, + warnings: ref seq[string] = nil): string = + ## If `error` is nil then no errors should be generated. + ## The same goes for `warnings`. + proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind, + arg: string) = + let mc = msgkind.whichMsgClass + let a = $msgkind % arg + var message: string + toLocation(message, filename, line, col + ColRstOffset) + message.add " $1: $2" % [$mc, a] + if mc == mcError: + doAssert error != nil, "unexpected RST error '" & message & "'" + error[] = message + # we check only first error because subsequent ones may be meaningless + raise newException(EParseError, message) + else: + doAssert warnings != nil, "unexpected RST warning '" & message & "'" + warnings[].add message + try: + result = rstToHtml(input, rstOptions, defaultConfig(), + msgHandler=testMsgHandler) + except EParseError: + discard suite "YAML syntax highlighting": test "Basics": @@ -24,7 +49,7 @@ suite "YAML syntax highlighting": ? key : value ...""" - let output = rstTohtml(input, {}, defaultConfig()) + let output = input.toHtml({}) doAssert output == """
%YAML 1.2
 ---
 a string: string
@@ -50,7 +75,7 @@ suite "YAML syntax highlighting":
     another literal block scalar:
       |+ # comment after header
      allowed, since more indented than parent"""
-    let output = rstToHtml(input, {}, defaultConfig())
+    let output = input.toHtml({})
     doAssert output == """
a literal block scalar: |
   some text
   # not a comment
@@ -76,7 +101,7 @@ suite "YAML syntax highlighting":
     % not a directive
     ...
     %TAG ! !foo:"""
-    let output = rstToHtml(input, {}, defaultConfig())
+    let output = input.toHtml({})
     doAssert output == """
%YAML 1.2
 ---
 %not a directive
@@ -97,7 +122,7 @@ suite "YAML syntax highlighting":
       more numbers: [-783, 11e78],
       not numbers: [ 42e, 0023, +32.37, 8 ball]
     }"""
-    let output = rstToHtml(input, {}, defaultConfig())
+    let output = input.toHtml({})
     doAssert output == """
{
   "quoted string": 42,
   'single quoted string': false,
@@ -114,7 +139,7 @@ suite "YAML syntax highlighting":
     : !localtag foo
     alias: *anchor
     """
-    let output = rstToHtml(input, {}, defaultConfig())
+    let output = input.toHtml({})
     doAssert output == """
--- !!map
 !!str string: !<tag:yaml.org,2002:int> 42
 ? &anchor !!seq []:
@@ -134,7 +159,7 @@ suite "YAML syntax highlighting":
     example.com/not/a#comment:
       ?not a map key
     """
-    let output = rstToHtml(input, {}, defaultConfig())
+    let output = input.toHtml({})
     doAssert output == """
...
  %a string:
   a:string:not:a:map
@@ -154,14 +179,12 @@ suite "RST/Markdown general":
       "Hello world!"
 
   test "Markdown links":
-    let
-      a = rstToHtml("(( [Nim](https://nim-lang.org/) ))", {roSupportMarkdown}, defaultConfig())
-      b = rstToHtml("(([Nim](https://nim-lang.org/)))", {roSupportMarkdown}, defaultConfig())
-      c = rstToHtml("[[Nim](https://nim-lang.org/)]", {roSupportMarkdown}, defaultConfig())
-
-    doAssert a == """(( Nim ))"""
-    doAssert b == """((Nim))"""
-    doAssert c == """[Nim]"""
+    check("(( [Nim](https://nim-lang.org/) ))".toHtml ==
+        """(( Nim ))""")
+    check("(([Nim](https://nim-lang.org/)))".toHtml ==
+        """((Nim))""")
+    check("[[Nim](https://nim-lang.org/)]".toHtml ==
+        """[Nim]""")
 
   test "Markdown tables":
     let input1 = """
@@ -172,7 +195,7 @@ suite "RST/Markdown general":
 | E1 \| text   |
 |              | F2 without pipe
 not in table"""
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     doAssert output1 == """
@@ -183,7 +206,7 @@ not in table"""
     let input2 = """
 | A1 header | A2 |
 | --- | --- |"""
-    let output2 = rstToHtml(input2, {roSupportMarkdown}, defaultConfig())
+    let output2 = input2.toHtml
     doAssert output2 == """
A1 headerA2 | not fooled

 
C1C2 bold

 
D1 code |D2
A1 headerA2

 
"""
 
@@ -225,7 +248,7 @@ A0     A1   X
     let input1 = """
 Check that a few punctuation symbols are not parsed as adornments:
 :word1: word2 .... word3 """
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     discard output1
 
   test "RST sections":
@@ -233,7 +256,7 @@ Check that a few punctuation symbols are not parsed as adornments:
 Long chapter name
 '''''''''''''''''''
 """
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     doAssert "Long chapter name" in output1 and "Some chapter" in output6
 
     # check that overline and underline match
@@ -287,16 +312,20 @@ Some chapter
       Some chapter
       -----------
       """
-    expect(EParseError):
-      let output7 = rstToHtml(input7, {roSupportMarkdown}, defaultConfig())
+    var error7 = new string
+    let output7 = input7.toHtml(error=error7)
+    check(error7[] == "input(1, 1) Error: new section expected (underline " &
+            "\'-----------\' does not match overline \'------------\')")
 
     let input8 = dedent """
       -----------
           Overflow
       -----------
       """
-    expect(EParseError):
-      let output8 = rstToHtml(input8, {roSupportMarkdown}, defaultConfig())
+    var error8 = new string
+    let output8 = input8.toHtml(error=error8)
+    check(error8[] == "input(1, 1) Error: new section expected (overline " &
+            "\'-----------\' is too short)")
 
     # check that hierarchy of title styles works
     let input9good = dedent """
@@ -319,7 +348,7 @@ Some chapter
       ~~~~~
 
       """
-    let output9good = rstToHtml(input9good, {roSupportMarkdown}, defaultConfig())
+    let output9good = input9good.toHtml
     doAssert "
Level1
" in output9good
     doAssert "
Level2
" in output9good
     doAssert "
Level3
" in output9good
@@ -348,8 +377,12 @@ Some chapter
       -------
 
       """
-    expect(EParseError):
-      let output9Bad = rstToHtml(input9Bad, {roSupportMarkdown}, defaultConfig())
+    var error9Bad = new string
+    let output9Bad = input9bad.toHtml(error=error9Bad)
+    check(error9Bad[] == "input(15, 1) Error: new section expected (section " &
+            "level inconsistent: underline ~~~~~ unexpectedly found, while " &
+            "the following intermediate section level(s) are missing on " &
+            "lines 12..15: underline -----)")
 
     # the same as input9good but with overline headings
     # first overline heading has a special meaning: document title
@@ -464,7 +497,7 @@ Some chapter
 Want to learn about `my favorite programming language`_?
 
 .. _my favorite programming language: https://nim-lang.org"""
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     doAssert "" in output1
 
   test "Markdown code block":
@@ -503,7 +538,7 @@ Test literal block
 ```
 let x = 1
 ``` """
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     doAssert "
 
Paragraph2
\n" == output2
 
     let input3 = dedent"""
@@ -563,7 +598,7 @@ Test1
       |   yyy
       |     zzz"""
 
-    let output3 = rstToHtml(input3, {roSupportMarkdown}, defaultConfig())
+    let output3 = input3.toHtml
     doAssert "xxx
" in output3
     doAssert "yyy
" in output3
     doAssert "zzz
" in output3
@@ -574,7 +609,7 @@ Test1
       |      
       |     zzz"""
 
-    let output4 = rstToHtml(input4, {roSupportMarkdown}, defaultConfig())
+    let output4 = input4.toHtml
     doAssert "xxx

" in output4
     doAssert "zzz
" in output4
 
@@ -597,7 +632,7 @@ Test1
       5. line5
          5
       """
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     for i in 1..5:
       doAssert ($i & ". line" & $i) notin output1
       doAssert ("
  • line" & $i & " " & $i & "
    • ") in output1
@@ -619,7 +654,7 @@ Test1
 
       8. line8
       """
-    let output2 = rstToHtml(input2, {roSupportMarkdown}, defaultConfig())
+    let output2 = input2.toHtml
     for i in [3, 4, 5, 7, 8]:
       doAssert ($i & ". line" & $i) notin output2
       doAssert ("
  • line" & $i & "
    • ") in output2
@@ -629,7 +664,7 @@ Test1
       1.  a) string1
       2. string2
       """
-    let output3 = rstToHtml(input3, {roSupportMarkdown}, defaultConfig())
+    let output3 = input3.toHtml
     doAssert count(output3, "
      ") == 2
     doAssert "
    1. string1
      2. " in output3 and "
    2. string2
      3. " in output3
@@ -645,7 +680,7 @@ Test1
       c) string5
       e) string6
       """
-    let output4 = rstToHtml(input4, {roSupportMarkdown}, defaultConfig())
+    let output4 = input4.toHtml
     doAssert count(output4, "
        ") == 4
     for enumerator in [9, 12]:
@@ -665,7 +700,7 @@ Test1
       #) string5
       #) string6
       """
-    let output5 = rstToHtml(input5, {roSupportMarkdown}, defaultConfig())
+    let output5 = input5.toHtml
     doAssert count(output5, "
          ") == 2
     doAssert count(output5, "
        1. ") == 5
@@ -677,7 +712,7 @@ Test1
       #. string2
       #. string3
       """
-    let output5a = rstToHtml(input5a, {roSupportMarkdown}, defaultConfig())
+    let output5a = input5a.toHtml
     doAssert count(output5a, "
            ") == 1
     doAssert count(output5a, "
          1. ") == 3
@@ -689,7 +724,7 @@ Test1
       #. string2
       #. string3
       """
-    let output6 = rstToHtml(input6, {roSupportMarkdown}, defaultConfig())
+    let output6 = input6.toHtml
     doAssert count(output6, "
              ") == 1
     doAssert count(output6, "
            1. ") == 3
@@ -702,7 +737,7 @@ Test1
       #. string2
       #. string3
       """
-    let output7 = rstToHtml(input7, {roSupportMarkdown}, defaultConfig())
+    let output7 = input7.toHtml
     doAssert count(output7, "
                ") == 1
     doAssert count(output7, "
              1. ") == 3
@@ -710,12 +745,20 @@ Test1
 
     # check that it's not recognized as enum.list without indentation on 2nd line
     let input8 = dedent """
-      A. string1
+      Paragraph.
+
+      A. stringA
+      B. stringB
+      C. string1
       string2
       """
-    # TODO: find out hot to catch warning here instead of throwing a defect
-    expect(AssertionDefect):
-      let output8 = input8.toHtml
+    var warnings8 = new seq[string]
+    let output8 = input8.toHtml(warnings = warnings8)
+    check(warnings8[].len == 1)
+    check("input(6, 1) Warning: RST style: \n" &
+          "  not enough indentation on line 6" in warnings8[0])
+    doAssert output8 == "Paragraph.
                  " &
+        "
                1. stringA
                  2. \n
                2. stringB
                  3. \n
                \n
                C. string1 string2 
                \n"
 
   test "Markdown enumerated lists":
     let input1 = dedent """
@@ -729,7 +772,7 @@ Test1
 
       #. lineA
       """
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     for i in 1..5:
       doAssert ($i & ". line" & $i) notin output1
       doAssert ("
              2. line" & $i & "
                3. ") in output1
@@ -755,7 +798,7 @@ Test1
       * line5
         5
       """
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     for i in 1..5:
       doAssert ("
              3. line" & $i & " " & $i & "
                4. ") in output1
     doAssert count(output1, "
                  Paragraph1
                  " in output0
 
   test "RST admonitions":
@@ -983,7 +1033,7 @@ Test1
       .. tip:: endOf tip
       .. warning:: endOf warning
     """
-    let output0 = rstToHtml(input0, {roSupportMarkdown}, defaultConfig())
+    let output0 = input0.toHtml
     for a in ["admonition", "attention", "caution", "danger", "error", "hint",
         "important", "note", "tip", "warning" ]:
       doAssert "endOf " & a & "" in output0
@@ -994,7 +1044,7 @@ Test1
 
       Test paragraph.
     """
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     doAssert "endOfError" in output1
     doAssert "
                  Test paragraph. 
                  " in output1
     doAssert "class=\"admonition admonition-error\"" in output1
@@ -1006,7 +1056,7 @@ Test1
 
       Test paragraph.
     """
-    let output2 = rstToHtml(input2, {roSupportMarkdown}, defaultConfig())
+    let output2 = input2.toHtml
     doAssert "endOfError Test2p." in output2
     doAssert "
                  Test paragraph. 
                  " in output2
     doAssert "class=\"admonition admonition-error\"" in output2
@@ -1014,7 +1064,7 @@ Test1
     let input3 = dedent """
       .. note:: endOfNote
     """
-    let output3 = rstToHtml(input3, {roSupportMarkdown}, defaultConfig())
+    let output3 = input3.toHtml
     doAssert "endOfNote" in output3
     doAssert "class=\"admonition admonition-info\"" in output3
 
@@ -1099,7 +1149,7 @@ Test1
 
       That was a transition.
     """
-    let output1 = rstToHtml(input1, {roSupportMarkdown}, defaultConfig())
+    let output1 = input1.toHtml
     doAssert "
                  some definition" in output1
     doAssert "Ref.