From 997a7009c1f3d03451710c0ea4f1777399f9371f Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Thu, 2 Apr 2020 18:54:20 +0300 Subject: [PATCH] WIP sourcemaps --- compiler/commands.nim | 4 + compiler/jsgen.nim | 43 +++-- compiler/options.nim | 2 + compiler/sourcemap.nim | 384 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 423 insertions(+), 10 deletions(-) create mode 100644 compiler/sourcemap.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index 4caccd1659a8b..babdd5871ae28 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -890,6 +890,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; processOnOffSwitchG(conf, {optPanics}, arg, pass, info) if optPanics in conf.globalOptions: defineSymbol(conf.symbols, "nimPanics") + of "sourcemap": + conf.globalOptions.incl optSourcemap + conf.options.incl optLineDir + # processOnOffSwitchG(conf, {optSourcemap, opt}, arg, pass, info) of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -) handleStdinInput(conf) else: diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index e63ce74895de8..da1d1bc018e8b 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -33,7 +33,7 @@ import nversion, msgs, idents, types, tables, ropes, math, passes, ccgutils, wordrecg, renderer, intsets, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, - transf, injectdestructors + transf, injectdestructors, sourcemap, json, sets from modulegraphs import ModuleGraph, PPassContext @@ -205,6 +205,8 @@ proc mapType(typ: PType): TJSTypeKind = proc mapType(p: PProc; typ: PType): TJSTypeKind = result = mapType(typ) +var mangled = initSet[string]() + proc mangleName(m: BModule, s: PSym): Rope = proc validJsName(name: string): bool = result = true @@ -259,6 +261,11 @@ proc mangleName(m: BModule, s: PSym): Rope = result.add(rope(s.id)) s.loc.r = result + # TODO: optimize + let s = $result + if '_' in s: + mangled.incl(s) + proc escapeJSString(s: string): string = result = newStringOfCap(s.len + s.len shr 2) result.add("\"") @@ -641,11 +648,16 @@ proc hasFrameInfo(p: PProc): bool = ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and ((p.prc == nil) or not (sfPure in p.prc.flags)) +proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope = + ropes.`%`("// line $2 \"$1\"$n", + [rope(toFullPath(config, info)), rope(line)]) + proc genLineDir(p: PProc, n: PNode) = let line = toLinenumber(n.info) - if optLineDir in p.options: - lineF(p, "// line $2 \"$1\"$n", - [rope(toFilename(p.config, n.info)), rope(line)]) + if line < 0: + return + if optLineDir in p.options or optLineDir in p.config.options: + lineF(p, "$1", [lineDir(p.config, n.info, line)]) if hasFrameInfo(p): lineF(p, "F.line = $1;$n", [rope(line)]) @@ -2241,6 +2253,10 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = p.nested: genStmt(p, transformedBody) + + if optLineDir in p.config.options: + result = lineDir(p.config, prc.info, toLinenumber(prc.info)) + var def: Rope if not prc.constraint.isNil: def = runtimeFormat(prc.constraint.strVal & " {$n$#$#$#$#$#", @@ -2253,7 +2269,8 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = optionalLine(genProcBody(p, prc)), optionalLine(p.indentLine(returnStmt))]) else: - result = ~"\L" + # if optLineDir in p.config.options: + # result.add(~"\L") if p.config.hcrOn: # Here, we introduce thunks that create the equivalent of a jump table @@ -2336,6 +2353,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = if r.kind != resCallee: r.kind = resNone #r.address = nil r.res = nil + case n.kind of nkSym: genSym(p, n, r) @@ -2382,14 +2400,15 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = else: r.res = rope(f.toStrMaxPrecision) r.kind = resExpr of nkCallKinds: - if isEmptyType(n.typ): genLineDir(p, n) + if isEmptyType(n.typ): + genLineDir(p, n) if (n[0].kind == nkSym) and (n[0].sym.magic != mNone): genMagic(p, n, r) elif n[0].kind == nkSym and sfInfixCall in n[0].sym.flags and n.len >= 1: genInfixCall(p, n, r) else: - genCall(p, n, r) + genCall(p, n, r) of nkClosure: gen(p, n[0], r) of nkCurly: genSetConstr(p, n, r) of nkBracket: genArrayConstr(p, n, r) @@ -2585,10 +2604,14 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = n.add destructorCall if passes.skipCodegen(m.config, n): return n if sfMainModule in m.module.flags: - let code = wholeCode(graph, m) + var code = genHeader() & wholeCode(graph, m) let outFile = m.config.prepareToWriteOutput() - discard writeRopeIfNotEqual(genHeader() & code, outFile) - + if optSourcemap in m.config.globalOptions: + var map: SourceMap + (code, map) = genSourceMap($(code), mangled, outFile.string) + writeFile(outFile.string & ".map", $(%map)) + discard writeRopeIfNotEqual(code, outFile) + proc myOpen(graph: ModuleGraph; s: PSym): PPassContext = result = newModule(graph, s) diff --git a/compiler/options.nim b/compiler/options.nim index 8b2523478b551..013c12b501a0b 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -44,6 +44,7 @@ type # please make sure we have under 32 options optOldAst, optSinkInference # 'sink T' inference + TOptions* = set[TOption] TGlobalOption* = enum # **keep binary compatible** gloptNone, optForceFullMake, @@ -94,6 +95,7 @@ type # please make sure we have under 32 options optProduceAsm # produce assembler code optPanics # turn panics (sysFatal) into a process termination optNimV1Emulation # emulate Nim v1.0 + optSourcemap TGlobalOptions* = set[TGlobalOption] diff --git a/compiler/sourcemap.nim b/compiler/sourcemap.nim new file mode 100644 index 0000000000000..919f1f95ed4c0 --- /dev/null +++ b/compiler/sourcemap.nim @@ -0,0 +1,384 @@ +import os, strformat, strutils, sequtils, tables, sets, ropes, json, algorithm + +type + SourceNode* = ref object + line*: int + column*: int + source*: string + name*: string + children*: seq[Child] + + C = enum cSourceNode, cString + + Child* = ref object + case kind*: C: + of cSourceNode: + node*: SourceNode + of cString: + s*: string + + SourceMap* = ref object + version*: int + sources*: seq[string] + names*: seq[string] + mappings*: string + file*: string + # sourceRoot*: string + # sourcesContent*: string + + SourceMapGenerator = ref object + file: string + sourceRoot: string + skipValidation: bool + sources: seq[string] + names: seq[string] + mappings: seq[Mapping] + + Mapping* = ref object + source*: string + original*: tuple[line: int, column: int] + generated*: tuple[line: int, column: int] + name*: string + noSource*: bool + noName*: bool + + +proc child*(s: string): Child = + Child(kind: cString, s: s) + + +proc child*(node: SourceNode): Child = + Child(kind: cSourceNode, node: node) + + +proc newSourceNode(line: int, column: int, path: string, node: SourceNode, name: string = ""): SourceNode = + SourceNode(line: line, column: column, source: path, name: name, children: @[child(node)]) + + +proc newSourceNode(line: int, column: int, path: string, s: string, name: string = ""): SourceNode = + SourceNode(line: line, column: column, source: path, name: name, children: @[child(s)]) + + +proc newSourceNode(line: int, column: int, path: string, children: seq[Child], name: string = ""): SourceNode = + SourceNode(line: line, column: column, source: path, name: name, children: children) + + + + +# debugging + + +proc text*(sourceNode: SourceNode, depth: int): string = + let empty = " " + result = &"{repeat(empty, depth)}SourceNode({sourceNode.source}:{sourceNode.line}:{sourceNode.column}):\n" + for child in sourceNode.children: + if child.kind == cString: + result.add(&"{repeat(empty, depth + 1)}{child.s}\n") + else: + result.add(child.node.text(depth + 1)) + + +proc `$`*(sourceNode: SourceNode): string = + text(sourceNode, 0) + + +# base64_VLQ + + +let integers = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" + + +proc encode*(i: int): string = + result = "" + var n = i + if n < 0: + n = (-n shl 1) or 1 + else: + n = n shl 1 + + var z = 0 + while z == 0 or n > 0: + var e = n and 31 + n = n shr 5 + if n > 0: + e = e or 32 + + result.add(integers[e]) + z += 1 + + +type + TokenState = enum Normal, String, Ident, Mangled + +proc tokenize*(line: string): seq[(bool, string)] = + result = @[] + var state = Normal + var token = "" + var isMangled = false + for z, ch in line: + if ch.isAlphaAscii: + if state == Normal: + state = Ident + if token.len > 0: + result.add((isMangled, token)) + token = $ch + isMangled = false + else: + token.add(ch) + elif ch == '_': + if state == Ident: + state = Mangled + isMangled = true + token.add($ch) + elif ch != '"' and not ch.isAlphaNumeric: + if state in {Ident, Mangled}: + state = Normal + if token.len > 0: + result.add((isMangled, token)) + token = $ch + isMangled = false + else: + token.add($ch) + elif ch == '"': + if state != String: + state = String + if token.len > 0: + result.add((isMangled, token)) + token = $ch + isMangled = false + else: + state = Normal + token.add($ch) + if token.len > 0: + result.add((isMangled, token)) + isMangled = false + token = "" + else: + token.add($ch) + if token.len > 0: + result.add((isMangled, token)) + +proc parse*(source: string, path: string, mangled: HashSet[string]): SourceNode = + let lines = source.splitLines() + var lastLocation: SourceNode = nil + result = newSourceNode(0, 0, path, @[]) + + # we just use one single parent and add all nim lines + # as its children, I guess in typical codegen + # that happens recursively on ast level + # we also don't have column info, but I doubt more one nim lines can compile to one js + # maybe in macros? + + for i, originalLine in lines: + let line = originalLine.strip + if line.len == 0: + continue + + # this shouldn't be a problem: + # jsgen doesn't generate comments + # and if you emit // line you probably know what you're doing + if line.startsWith("// line"): + if result.children.len > 0: + result.children[^1].node.children.add(child(line & "\n")) + let pos = line.find(" ", 8) + let lineNumber = line[8 .. pos - 1].parseInt + let linePath = line[pos + 2 .. ^2] # quotes + # echo "line ", lineNumber, " ", linePath + lastLocation = newSourceNode( + lineNumber, + 0, + linePath, + @[]) + result.children.add(child(lastLocation)) + else: + let tokens = line.tokenize() + for j, token in tokens: + var name = "" + if token[0]: + name = token[1].split('_', 1)[0] + + let nl = if j == tokens.len - 1: "\n" else: "" + if result.children.len > 0: + result.children[^1].node.children.add( + child( + newSourceNode( + result.children[^1].node.line, + 0, + result.children[^1].node.source, + (token[1] & nl), + name))) + else: + result.children.add( + child( + newSourceNode(i + 1, 0, path, token[1] & nl, name))) + +proc cmp(a: Mapping, b: Mapping): int = + var c = cmp(a.generated, b.generated) + if c != 0: + return c + + c = cmp(a.source, b.source) + if c != 0: + return c + + c = cmp(a.original, b.original) + if c != 0: + return c + + return cmp(a.name, b.name) + + +proc index*[T](elements: seq[T], element: T): int = + for z in 0 ..< elements.len: + if elements[z] == element: + return z + return -1 + + +proc serializeMappings(map: SourceMapGenerator, mappings: seq[Mapping]): string = + var previous = Mapping(generated: (line: 1, column: 0), original: (line: 0, column: 0), name: "", source: "") + var previousSourceId = 0 + var previousNameId = 0 + result = "" + var next = "" + var nameId = 0 + var sourceId = 0 + + for z, mapping in mappings: + next = "" + + if mapping.generated.line != previous.generated.line: + previous.generated.column = 0 + + while mapping.generated.line != previous.generated.line: + next.add(";") + previous.generated.line += 1 + + else: + if z > 0: + if cmp(mapping, mappings[z - 1]) == 0: + continue + next.add(",") + + next.add(encode(mapping.generated.column - previous.generated.column)) + previous.generated.column = mapping.generated.column + + if not mapping.noSource and mapping.source.len > 0: + sourceId = map.sources.index(mapping.source) + next.add(encode(sourceId - previousSourceId)) + previousSourceId = sourceId + next.add(encode(mapping.original.line - 1 - previous.original.line)) + previous.original.line = mapping.original.line - 1 + next.add(encode(mapping.original.column - previous.original.column)) + previous.original.column = mapping.original.column + + if not mapping.noName and mapping.name.len > 0: + nameId = map.names.index(mapping.name) + next.add(encode(nameId - previousNameId)) + previousNameId = nameId + + result.add(next) + + +proc gen*(map: SourceMapGenerator): SourceMap = + var mappings = map.mappings.sorted do (a: Mapping, b: Mapping) -> int: + cmp(a, b) + result = SourceMap( + file: map.file, + version: 3, + sources: map.sources[0..^1], + names: map.names[0..^1], + mappings: map.serializeMappings(mappings)) + + +# TODO: optimize + + +proc addMapping*(map: SourceMapGenerator, mapping: Mapping) = + if not mapping.noSource and mapping.source notin map.sources: + map.sources.add(mapping.source) + + if not mapping.noName and mapping.name.len > 0 and mapping.name notin map.names: + map.names.add(mapping.name) + + # echo "map ", mapping.source, " ", mapping.original, " ", mapping.generated, " ", mapping.name + map.mappings.add(mapping) + + +proc walk*(node: SourceNode, fn: proc(line: string, original: SourceNode)) = + for child in node.children: + if child.kind == cString and child.s.len > 0: + fn(child.s, node) + else: + child.node.walk(fn) + + +proc toStringWithSourceMap*(node: SourceNode, file: string): tuple[code: string, map: SourceMapGenerator] = + var code = "" + var map = SourceMapGenerator(file: file, sources: @[], names: @[], mappings: @[]) + + var generated = (line: 1, column: 0) + var sourceMappingActive = false + var lastOriginal = SourceNode(source: "", line: -1, column: 0, name: "", children: @[]) + + node.walk do (line: string, original: SourceNode): + code.add(line) + if original.source.endsWith(".js"): + # ignore it + discard + else: + if original.line != -1: + if lastOriginal.source != original.source or + lastOriginal.line != original.line or + lastOriginal.column != original.column or + lastOriginal.name != original.name: + map.addMapping( + Mapping( + source: original.source, + original: (line: original.line, column: original.column), + generated: (line: generated.line, column: generated.column), + name: original.name)) + + lastOriginal = SourceNode( + source: original.source, + line: original.line, + column: original.column, + name: original.name, + children: lastOriginal.children) + sourceMappingActive = true + elif sourceMappingActive: + map.addMapping( + Mapping( + noSource: true, + noName: true, + generated: (line: generated.line, column: generated.column), + original: (line: -1, column: -1))) + lastOriginal.line = -1 + sourceMappingActive = false + + for z in 0 ..< line.len: + if line[z] in Newlines: + generated.line += 1 + generated.column = 0 + + if z == line.len - 1: + lastOriginal.line = -1 + sourceMappingActive = false + elif sourceMappingActive: + map.addMapping( + Mapping( + source: original.source, + original: (line: original.line, column: original.column), + generated: (line: generated.line, column: generated.column), + name: original.name)) + else: + generated.column += 1 + + (code: code, map: map) + + +proc genSourceMap*(source: string, mangled: HashSet[string], outfile: string): (Rope, SourceMap) = + let node = parse(source, outfile, mangled) + let map = node.toStringWithSourceMap(file = outfile) + ((&"{map.code}\n//# sourceMappingURL={outfile}.map").rope, map.map.gen) +