WIP: use dedent for de-indentation in lexer, fix #15184

compiler/lexer.nim

@@ -950,18 +950,10 @@ proc newlineFollows*(L: Lexer): bool =
 proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
                           isDoc: bool) =
   var pos = start
-  var toStrip = 0
   tokenBegin(tok, pos)
-  # detect the amount of indentation:
-  if isDoc:
-    toStrip = getColNumber(L, pos)
-    while L.buf[pos] == ' ': inc pos
-    if L.buf[pos] in {CR, LF}:
-      pos = handleCRLF(L, pos)
-      toStrip = 0
-      while L.buf[pos] == ' ':
-        inc pos
-        inc toStrip
+  if isDoc and getColNumber(L, pos) > "##[".len:
+    # the next text is indented as if it's placed at the beginning of ##[
+    tok.literal.add " ".repeat(getColNumber(L, pos) - "##[".len)
   var nesting = 0
   while true:
     case L.buf[pos]
@@ -997,10 +989,6 @@ proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
       if isDoc:
         when not defined(nimpretty): tok.literal.add "\n"
         inc tok.iNumber
-        var c = toStrip
-        while L.buf[pos] == ' ' and c > 0:
-          inc pos
-          dec c
     of nimlexbase.EndOfFile:
       tokenEndIgnore(tok, pos)
       lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
@@ -1011,6 +999,8 @@ proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
   L.bufpos = pos
   when defined(nimpretty):
     tok.commentOffsetB = L.offsetBase + pos - 1
+  if isDoc:
+    tok.literal = dedent(tok.literal)
 
 proc scanComment(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
@@ -1027,11 +1017,6 @@ proc scanComment(L: var Lexer, tok: var Token) =
   tokenBegin(tok, pos)
   inc(pos, 2)
 
-  var toStrip = 0
-  while L.buf[pos] == ' ':
-    inc pos
-    inc toStrip
-
   while true:
     var lastBackslash = -1
     while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
@@ -1048,10 +1033,6 @@ proc scanComment(L: var Lexer, tok: var Token) =
     if L.buf[pos] == '#' and L.buf[pos+1] == '#':
       tok.literal.add "\n"
       inc(pos, 2)
-      var c = toStrip
-      while L.buf[pos] == ' ' and c > 0:
-        inc pos
-        dec c
       inc tok.iNumber
     else:
       if L.buf[pos] > ' ':
@@ -1061,6 +1042,7 @@ proc scanComment(L: var Lexer, tok: var Token) =
   L.bufpos = pos
   when defined(nimpretty):
     tok.commentOffsetB = L.offsetBase + pos - 1
+  tok.literal = dedent(tok.literal)
 
 proc skip(L: var Lexer, tok: var Token) =
   var pos = L.bufpos

lib/pure/strutils.nim

@@ -1439,43 +1439,50 @@ func unindent*(s: string, count: Natural = int.high,
     result.add(line[indentCount*padding.len .. ^1])
     i.inc
 
-func indentation*(s: string): Natural {.since: (1, 3).} =
-  ## Returns the amount of indentation all lines of `s` have in common,
-  ## ignoring lines that consist only of whitespace.
-  result = int.high
-  for line in s.splitLines:
-    for i, c in line:
-      if i >= result: break
-      elif c != ' ':
-        result = i
-        break
-  if result == int.high:
-    result = 0
-
-func dedent*(s: string, count: Natural = indentation(s)): string {.rtl,
-    extern: "nsuDedent", since: (1, 3).} =
-  ## Unindents each line in `s` by `count` amount of `padding`.
-  ## The only difference between this and the
-  ## `unindent func<#unindent,string,Natural,string>`_ is that this by default
-  ## only cuts off the amount of indentation that all lines of `s` share as
-  ## opposed to all indentation. It only supports spaces as padding.
-  ##
-  ## **Note:** This does not preserve the new line characters used in `s`.
-  ##
-  ## See also:
-  ## * `unindent func<#unindent,string,Natural,string>`_
-  ## * `align func<#align,string,Natural,char>`_
-  ## * `alignLeft func<#alignLeft,string,Natural,char>`_
-  ## * `spaces func<#spaces,Natural>`_
-  ## * `indent func<#indent,string,Natural,string>`_
-  runnableExamples:
-    let x = """
-      Hello
-        There
-    """.dedent()
-
-    doAssert x == "Hello\n  There\n"
-  unindent(s, count, " ")
+since (1,3):
+  func indentation*(s: string): Natural =
+    ## Returns the amount of indentation all lines of `s` have in common,
+    ## ignoring lines that consist only of whitespace.
+    result = int.high
+    for line in s.splitLines:
+      for i, c in line:
+        if i >= result: break
+        elif c != ' ':
+          result = i
+          break
+    if result == int.high:
+      result = 0
+
+  func dedent*(s: string, count: Natural): string {.rtl,
+      extern: "nsuDedent".} =
+    ## Unindents each line in `s` by `count` amount of `padding`.
+    ## The only difference between this and the
+    ## `unindent func<#unindent,string,Natural,string>`_ is that this by default
+    ## only cuts off the amount of indentation that all lines of `s` share as
+    ## opposed to all indentation. It only supports spaces as padding.
+    ##
+    ## **Note:** This does not preserve the new line characters used in `s`.
+    ##
+    ## See also:
+    ## * `unindent func<#unindent,string,Natural,string>`_
+    ## * `align func<#align,string,Natural,char>`_
+    ## * `alignLeft func<#alignLeft,string,Natural,char>`_
+    ## * `spaces func<#spaces,Natural>`_
+    ## * `indent func<#indent,string,Natural,string>`_
+    runnableExamples:
+      let x = """
+        Hello
+          There
+      """.dedent()
+
+      doAssert x == "Hello\n  There\n"
+    result = unindent(s, count, " ")
+
+  proc dedent*(a: string): string =
+    # pending https://github.com/timotheecour/Nim/issues/521
+    let b = a
+    let i = indentation(b)
+    dedent(a, i)
 
 func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete".} =
   ## Deletes in `s` (must be declared as `var`) the characters at positions