From 73073d571bd56dd4c5673a188cd598a4b0ae3b49 Mon Sep 17 00:00:00 2001 From: Alexander Ivanov Date: Sat, 14 Apr 2018 10:27:01 +0300 Subject: [PATCH] Sourcemaps for JS --- compiler/commands.nim | 3 + compiler/jsgen.nim | 34 +++- compiler/options.nim | 4 +- compiler/sourcemap.nim | 377 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 412 insertions(+), 6 deletions(-) create mode 100644 compiler/sourcemap.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index 39967c4bc3ac3..c345e802802a0 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -290,6 +290,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool of "patterns": result = contains(conf.options, optPatterns) of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace) of "nilseqs": result = contains(conf.options, optNilSeqs) + of "sourcemap": result = contains(conf.globalOptions, optSourceMap) else: invalidCmdLineOption(conf, passCmd1, switch, info) proc processPath(conf: ConfigRef; path: string, info: TLineInfo, @@ -740,6 +741,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; else: conf.cppCustomNamespace = "Nim" defineSymbol(conf.symbols, "cppCompileToNamespace", conf.cppCustomNamespace) + of "sourcemap": + incl(conf.globalOptions, optSourceMap) else: if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg) else: invalidCmdLineOption(conf, pass, switch, info) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index aa23865260c99..a2dca8a9de512 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -32,7 +32,7 @@ import ast, astalgo, strutils, hashes, trees, platform, magicsys, extccomp, options, nversion, nimsets, msgs, std / sha1, bitsets, idents, types, os, tables, times, ropes, math, passes, ccgutils, wordrecg, renderer, - intsets, cgmeth, lowerings, sighashes, lineinfos, rodutils, pathutils + intsets, cgmeth, lowerings, sighashes, lineinfos, rodutils, pathutils, sets, sourcemap, json from modulegraphs import ModuleGraph @@ -200,6 +200,8 @@ proc mapType(typ: PType): TJSTypeKind = proc mapType(p: PProc; typ: PType): TJSTypeKind = result = mapType(typ) +var mangled = initSet[string]() + proc mangleName(m: BModule, s: PSym): Rope = proc validJsName(name: string): bool = result = true @@ -254,6 +256,11 @@ proc mangleName(m: BModule, s: PSym): Rope = add(result, rope(s.id)) s.loc.r = result + # TODO: optimize + let s = $result + if '_' in s: + mangled.incl(s) + proc escapeJSString(s: string): string = result = newStringOfCap(s.len + s.len shr 2) result.add("\"") @@ -532,11 +539,17 @@ proc hasFrameInfo(p: PProc): bool = ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and ((p.prc == nil) or not (sfPure in p.prc.flags)) +proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope = + ropes.`%`("$N// line $2 $1$N", + [rope(toFilename(config, info)), rope(line)]) + proc genLineDir(p: PProc, n: PNode) = let line = toLinenumber(n.info) + if line < 0: + return if optLineDir in p.options: - lineF(p, "// line $2 \"$1\"$n", - [rope(toFilename(p.config, n.info)), rope(line)]) + # softRnl = noRnl + line(p, lineDir(p.config, n.info, line)) if {optStackTrace, optEndb} * p.options == {optStackTrace, optEndb} and ((p.prc == nil) or sfPure notin p.prc.flags): useMagic(p, "endb") @@ -1961,6 +1974,10 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = p.nested: genStmt(p, prc.getBody) + + if optLineDir in p.config.options: + result = lineDir(p.config, prc.info, toLinenumber(prc.info)) + var def: Rope if not prc.constraint.isNil: def = (prc.constraint.strVal & " {$n$#$#$#$#$#") % @@ -1973,7 +1990,8 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = optionaLine(genProcBody(p, prc)), optionaLine(p.indentLine(returnStmt))] else: - result = ~"\L" + if optLineDir in p.config.options: + result = ~"\L" if optHotCodeReloading in p.config.options: # Here, we introduce thunks that create the equivalent of a jump table @@ -2275,14 +2293,20 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = let ext = "js" let f = if globals.classes.len == 0: toFilename(m.config, FileIndex m.module.position) else: "nimsystem" - let code = wholeCode(graph, m) + var code = wholeCode(graph, m) let outfile = if not m.config.outFile.isEmpty: if m.config.outFile.string.isAbsolute: m.config.outFile else: AbsoluteFile(getCurrentDir() / m.config.outFile.string) else: changeFileExt(completeCFilePath(m.config, AbsoluteFile f), ext) + if optSourceMap in m.config.globalOptions: + var map: SourceMap + (code, map) = genSourceMap($code, mangled, f, outfile.string) + writeFile(outfile.string & ".map", $(%map)) + discard writeRopeIfNotEqual(genHeader() & code, outfile) + for obj, content in items(globals.classes): genClass(m.config, obj, content, ext) diff --git a/compiler/options.nim b/compiler/options.nim index b4d2bb64e1113..defb287506e18 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -42,6 +42,7 @@ type # please make sure we have under 32 options optLaxStrings, optNilSeqs + TOptions* = set[TOption] TGlobalOption* = enum # **keep binary compatible** gloptNone, optForceFullMake, @@ -77,7 +78,8 @@ type # please make sure we have under 32 options optMixedMode # true if some module triggered C++ codegen optListFullPaths optNoNimblePath - optDynlibOverrideAll + optDynlibOverrideAll, + optSourcemap TGlobalOptions* = set[TGlobalOption] diff --git a/compiler/sourcemap.nim b/compiler/sourcemap.nim new file mode 100644 index 0000000000000..2ff4a4cad9390 --- /dev/null +++ b/compiler/sourcemap.nim @@ -0,0 +1,377 @@ +import os, strformat, strutils, sequtils, tables, sets, ropes, json, algorithm + +type + SourceNode* = ref object + line*: int + column*: int + source*: string + name*: string + children*: seq[Child] + + C = enum cSourceNode, cString + + Child* = ref object + case kind*: C: + of cSourceNode: + node*: SourceNode + of cString: + s*: string + + SourceMap* = ref object + version*: int + sources*: seq[string] + names*: seq[string] + mappings*: string + file*: string + # sourceRoot*: string + # sourcesContent*: string + + SourceMapGenerator = ref object + file: string + sourceRoot: string + skipValidation: bool + sources: seq[string] + names: seq[string] + mappings: seq[Mapping] + + Mapping* = ref object + source*: string + original*: tuple[line: int, column: int] + generated*: tuple[line: int, column: int] + name*: string + noSource*: bool + noName*: bool + + +proc child*(s: string): Child = + Child(kind: cString, s: s) + + +proc child*(node: SourceNode): Child = + Child(kind: cSourceNode, node: node) + + +proc newSourceNode(line: int, column: int, path: string, node: SourceNode, name: string = ""): SourceNode = + SourceNode(line: line, column: column, source: path, name: name, children: @[child(node)]) + + +proc newSourceNode(line: int, column: int, path: string, s: string, name: string = ""): SourceNode = + SourceNode(line: line, column: column, source: path, name: name, children: @[child(s)]) + + +proc newSourceNode(line: int, column: int, path: string, children: seq[Child], name: string = ""): SourceNode = + SourceNode(line: line, column: column, source: path, name: name, children: children) + + + + +# debugging + + +proc text*(sourceNode: SourceNode, depth: int): string = + let empty = " " + result = &"{repeat(empty, depth)}SourceNode({sourceNode.source}:{sourceNode.line}:{sourceNode.column}):\n" + for child in sourceNode.children: + if child.kind == cString: + result.add(&"{repeat(empty, depth + 1)}{child.s}\n") + else: + result.add(child.node.text(depth + 1)) + + +proc `$`*(sourceNode: SourceNode): string = + text(sourceNode, 0) + + +# base64_VLQ + + +let integers = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" + + +proc encode*(i: int): string = + result = "" + var n = i + if n < 0: + n = (-n shl 1) or 1 + else: + n = n shl 1 + + var z = 0 + while z == 0 or n > 0: + var e = n and 31 + n = n shr 5 + if n > 0: + e = e or 32 + + result.add(integers[e]) + z += 1 + + +type + TokenState = enum Normal, String, Ident, Mangled + +proc tokenize*(line: string): seq[(bool, string)] = + result = @[] + var state = Normal + var token = "" + var isMangled = false + for z, ch in line: + if ch.isAlphaAscii: + if state == Normal: + state = Ident + if token.len > 0: + result.add((isMangled, token)) + token = $ch + isMangled = false + else: + token.add(ch) + elif ch == '_': + if state == Ident: + state = Mangled + isMangled = true + token.add($ch) + elif ch != '"' and not ch.isAlphaNumeric: + if state in {Ident, Mangled}: + state = Normal + if token.len > 0: + result.add((isMangled, token)) + token = $ch + isMangled = false + else: + token.add($ch) + elif ch == '"': + if state != String: + state = String + if token.len > 0: + result.add((isMangled, token)) + token = $ch + isMangled = false + else: + state = Normal + token.add($ch) + if token.len > 0: + result.add((isMangled, token)) + isMangled = false + token = "" + else: + token.add($ch) + if token.len > 0: + result.add((isMangled, token)) + +proc parse*(source: string, path: string, mangled: HashSet[string]): SourceNode = + let lines = source.splitLines() + var lastLocation: SourceNode = nil + result = newSourceNode(0, 0, path, @[]) + + # we just use one single parent and add all nim lines + # as its children, I guess in typical codegen + # that happens recursively on ast level + # we also don't have column info, but I doubt more one nim lines can compile to one js + + for i, originalLine in lines: + let line = originalLine.strip + if line.len == 0: + continue + + # this shouldn't be a problem: + # jsgen doesn't generate comments + # and if you emit // line you probably know what you're doing + if line.startsWith("// line"): + if result.children.len > 0: + result.children[^1].node.children.add(child(line & "\n")) + let pos = line.find(" ", 8) + let lineNumber = line[8 .. pos - 1].parseInt + let linePath = line[pos + 1 .. ^1] # quotes + lastLocation = newSourceNode( + lineNumber, + 0, + linePath, + @[]) + result.children.add(child(lastLocation)) + else: + let tokens = line.tokenize() + for j, token in tokens: + var name = "" + if token[0]: + name = token[1].split('_', 1)[0] + + let nl = if j == tokens.len - 1: "\n" else: "" + if result.children.len > 0: + result.children[^1].node.children.add( + child( + newSourceNode( + result.children[^1].node.line, + 0, + result.children[^1].node.source, + (token[1] & nl), + name))) + else: + result.children.add( + child( + newSourceNode(i + 1, 0, path, token[1] & nl, name))) + +proc cmp(a: Mapping, b: Mapping): int = + var c = cmp(a.generated, b.generated) + if c != 0: + return c + + c = cmp(a.source, b.source) + if c != 0: + return c + + c = cmp(a.original, b.original) + if c != 0: + return c + + return cmp(a.name, b.name) + + +proc index*[T](elements: seq[T], element: T): int = + for z in 0 ..< elements.len: + if elements[z] == element: + return z + return -1 + + +proc serializeMappings(map: SourceMapGenerator, mappings: seq[Mapping]): string = + var previous = Mapping(generated: (line: 1, column: 0), original: (line: 0, column: 0), name: "", source: "") + var previousSourceId = 0 + var previousNameId = 0 + result = "" + var next = "" + var nameId = 0 + var sourceId = 0 + + for z, mapping in mappings: + next = "" + + if mapping.generated.line != previous.generated.line: + previous.generated.column = 0 + + while mapping.generated.line != previous.generated.line: + next.add(";") + previous.generated.line += 1 + + else: + if z > 0: + if cmp(mapping, mappings[z - 1]) == 0: + continue + next.add(",") + + next.add(encode(mapping.generated.column - previous.generated.column)) + previous.generated.column = mapping.generated.column + + if not mapping.noSource and mapping.source.len > 0: + sourceId = map.sources.index(mapping.source) + next.add(encode(sourceId - previousSourceId)) + previousSourceId = sourceId + next.add(encode(mapping.original.line - 1 - previous.original.line)) + previous.original.line = mapping.original.line - 1 + next.add(encode(mapping.original.column - previous.original.column)) + previous.original.column = mapping.original.column + + if not mapping.noName and mapping.name.len > 0: + nameId = map.names.index(mapping.name) + next.add(encode(nameId - previousNameId)) + previousNameId = nameId + + result.add(next) + + +proc gen*(map: SourceMapGenerator): SourceMap = + var mappings = map.mappings.sorted do (a: Mapping, b: Mapping) -> int: + cmp(a, b) + result = SourceMap( + file: map.file, + version: 3, + sources: map.sources[0..^1], + names: map.names[0..^1], + mappings: map.serializeMappings(mappings)) + + +# TODO: optimize + + +proc addMapping*(map: SourceMapGenerator, mapping: Mapping) = + if not mapping.noSource and mapping.source notin map.sources: + map.sources.add(mapping.source) + + if not mapping.noName and mapping.name.len > 0 and mapping.name notin map.names: + map.names.add(mapping.name) + + map.mappings.add(mapping) + + +proc walk*(node: SourceNode, fn: proc(line: string, original: SourceNode)) = + for child in node.children: + if child.kind == cString and child.s.len > 0: + fn(child.s, node) + else: + child.node.walk(fn) + + +proc toStringWithSourceMap*(node: SourceNode, file: string): tuple[code: string, map: SourceMapGenerator] = + var code = "" + var map = SourceMapGenerator(file: file, sources: @[], names: @[], mappings: @[]) + + var generated = (line: 1, column: 0) + var sourceMappingActive = false + var lastOriginal = SourceNode(source: "", line: -1, column: 0, name: "", children: @[]) + + node.walk do (line: string, original: SourceNode): + code.add(line) + if original.line != -1: + if lastOriginal.source != original.source or + lastOriginal.line != original.line or + lastOriginal.column != original.column or + lastOriginal.name != original.name: + map.addMapping( + Mapping( + source: original.source, + original: (line: original.line, column: original.column), + generated: (line: generated.line, column: generated.column), + name: original.name)) + + lastOriginal = SourceNode( + source: original.source, + line: original.line, + column: original.column, + name: original.name, + children: lastOriginal.children) + sourceMappingActive = true + elif sourceMappingActive: + map.addMapping( + Mapping( + noSource: true, + noName: true, + generated: (line: generated.line, column: generated.column), + original: (line: -1, column: -1))) + lastOriginal.line = -1 + sourceMappingActive = false + + for z in 0 ..< line.len: + if line[z] in NewLines: + generated.line += 1 + generated.column = 0 + + if z == line.len - 1: + lastOriginal.line = -1 + sourceMappingActive = false + elif sourceMappingActive: + map.addMapping( + Mapping( + source: original.source, + original: (line: original.line, column: original.column), + generated: (line: generated.line, column: generated.column), + name: original.name)) + else: + generated.column += 1 + + (code: code, map: map) + + +proc genSourceMap*(source: string, mangled: HashSet[string], path: string, outfile: string): (Rope, SourceMap) = + let node = parse(source, outfile, mangled) + let map = node.toStringWithSourceMap(file = outfile) + ((&"{map.code}\n//# sourceMappingURL={outfile}.map").rope, map.map.gen) +