preparations for IC+ORC

compiler/ast.nim

@@ -854,7 +854,7 @@ type
     of routineKinds:
       #procInstCache*: seq[PInstantiation]
       gcUnsafetyReason*: PSym  # for better error messages wrt gcsafe
-      transformedBody*: PNode  # cached body after transf pass
+      semcheckedBody*: PNode   # proc body after semantic checking
     of skLet, skVar, skField, skForVar:
       guard*: PSym
       bitsize*: int
@@ -1636,7 +1636,8 @@ proc transitionGenericParamToType*(s: PSym) =
 proc transitionRoutineSymKind*(s: PSym, kind: range[skProc..skTemplate]) =
   transitionSymKindCommon(kind)
   s.gcUnsafetyReason = obj.gcUnsafetyReason
-  s.transformedBody = obj.transformedBody
+  s.semcheckedBody = obj.semcheckedBody
+  #s.transformedBody = obj.transformedBody
 
 proc transitionToLet*(s: PSym) =
   transitionSymKindCommon(skLet)

compiler/cgen.nim

@@ -1014,9 +1014,11 @@ proc genProcAux(m: BModule, prc: PSym) =
   var returnStmt: Rope = nil
   assert(prc.ast != nil)
 
-  var procBody = transformBody(m.g.graph, m.idgen, prc, cache = false)
-  if sfInjectDestructors in prc.flags:
-    procBody = injectDestructorCalls(m.g.graph, m.idgen, prc, procBody)
+  when false:
+    var procBody = transformBody(m.g.graph, m.idgen, prc, cache = false)
+    if sfInjectDestructors in prc.flags:
+      procBody = injectDestructorCalls(m.g.graph, m.idgen, prc, procBody)
+  let procBody = getBody(m.g.graph, prc)
 
   if sfPure notin prc.flags and prc.typ[0] != nil:
     if resultPos >= prc.ast.len:

compiler/cgmeth.nim

@@ -11,47 +11,7 @@
 
 import
   intsets, options, ast, msgs, idents, renderer, types, magicsys,
-  sempass2, strutils, modulegraphs, lineinfos
-
-proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
-  var dest = skipTypes(d, abstractPtrs)
-  var source = skipTypes(n.typ, abstractPtrs)
-  if (source.kind == tyObject) and (dest.kind == tyObject):
-    var diff = inheritanceDiff(dest, source)
-    if diff == high(int):
-      # no subtype relation, nothing to do
-      result = n
-    elif diff < 0:
-      result = newNodeIT(nkObjUpConv, n.info, d)
-      result.add n
-      if downcast: internalError(conf, n.info, "cgmeth.genConv: no upcast allowed")
-    elif diff > 0:
-      result = newNodeIT(nkObjDownConv, n.info, d)
-      result.add n
-      if not downcast:
-        internalError(conf, n.info, "cgmeth.genConv: no downcast allowed")
-    else:
-      result = n
-  else:
-    result = n
-
-proc getDispatcher*(s: PSym): PSym =
-  ## can return nil if is has no dispatcher.
-  if dispatcherPos < s.ast.len:
-    result = s.ast[dispatcherPos].sym
-    doAssert sfDispatcher in result.flags
-
-proc methodCall*(n: PNode; conf: ConfigRef): PNode =
-  result = n
-  # replace ordinary method by dispatcher method:
-  let disp = getDispatcher(result[0].sym)
-  if disp != nil:
-    result[0].sym = disp
-    # change the arguments to up/downcasts to fit the dispatcher's parameters:
-    for i in 1..<result.len:
-      result[i] = genConv(result[i], disp.typ[i], true, conf)
-  else:
-    localError(conf, n.info, "'" & $result[0] & "' lacks a dispatcher")
+  sempass2, strutils, modulegraphs, lineinfos, transf
 
 type
   MethodResult = enum No, Invalid, Yes

compiler/enumtostr.nim

@@ -26,7 +26,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGener
     assert(t.n[i].kind == nkSym)
     var field = t.n[i].sym
     let val = if field.ast == nil: field.name.s else: field.ast.strVal
-    caseStmt.add newTree(nkOfBranch, newSymNode(field),
+    caseStmt.add newTree(nkOfBranch, newIntTypeNode(field.position, t),
       newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newStrNode(val, info))))
     #newIntTypeNode(nkIntLit, field.position, t)
 

compiler/finalast.nim

@@ -0,0 +1,22 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module implements a tiny intermediate layer.
+
+import
+  ast, transf, injectdestructors, modulegraphs
+
+proc finalProcBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; n: PNode): PNode =
+  ## Transformations after sem'checking that we need to do.
+  result = transformBody(g, idgen, prc, dontUseCache)
+  if sfInjectDestructors in prc.flags:
+    result = injectDestructorCalls(g, idgen, prc, result)
+
+proc finalToplevelStmt*(g: ModuleGraph; idgen: IdGenerator; module: PSym; n: PNode): PNode =
+  result = transformStmt(g, idgen, module, n)

compiler/ic/ic.nim

@@ -34,17 +34,15 @@ type
     toReplay*: PackedTree # pragmas and VM specific state to replay.
     topLevel*: PackedTree  # top level statements
     bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
-    #producedGenerics*: Table[GenericKey, SymId]
     exports*: seq[(LitId, int32)]
     hidden*: seq[(LitId, int32)]
     reexports*: seq[(LitId, PackedItemId)]
     compilerProcs*: seq[(LitId, int32)]
     converters*, methods*, trmacros*, pureEnums*: seq[int32]
-    macroUsages*: seq[(PackedItemId, PackedLineInfo)]
 
     typeInstCache*: seq[(PackedItemId, PackedItemId)]
     procInstCache*: seq[PackedInstantiation]
-    attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)]
+    attachedOps*: seq[(PackedItemId, TTypeAttachedOp, PackedItemId)]
     methodsPerType*: seq[(PackedItemId, int, PackedItemId)]
     enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
 
@@ -523,6 +521,15 @@ proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var Packe
   toPackedProcDef(s.ast, m.topLevel, encoder, m)
   #flush encoder, m
 
+proc storeAttachedProcDef*(t: PType; op: TTypeAttachedOp; s: PSym,
+                           encoder: var PackedEncoder; m: var PackedModule) =
+  assert s.kind in routineKinds
+  assert isActive(encoder)
+  let tid = storeTypeLater(t, encoder, m)
+  let sid = storeSymLater(s, encoder, m)
+  m.attachedOps.add (tid, op, sid)
+  toPackedGeneratedProcDef(s, encoder, m)
+
 proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
   var t = newSeq[PackedItemId](i.concreteTypes.len)
   for j in 0..high(i.concreteTypes):
@@ -587,7 +594,6 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadSeqSection convertersSection, m.converters
   loadSeqSection methodsSection, m.methods
   loadSeqSection pureEnumsSection, m.pureEnums
-  loadSeqSection macroUsagesSection, m.macroUsages
 
   loadSeqSection toReplaySection, m.toReplay.nodes
   loadSeqSection topLevelSection, m.topLevel.nodes
@@ -651,7 +657,6 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection convertersSection, m.converters
   storeSeqSection methodsSection, m.methods
   storeSeqSection pureEnumsSection, m.pureEnums
-  storeSeqSection macroUsagesSection, m.macroUsages
 
   storeSeqSection toReplaySection, m.toReplay.nodes
   storeSeqSection topLevelSection, m.topLevel.nodes
@@ -892,16 +897,10 @@ proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
                         t: PackedType; si, item: int32; result: PType) =
   result.sym = loadSym(c, g, si, t.sym)
   result.owner = loadSym(c, g, si, t.owner)
-  when false:
-    for op, item in pairs t.attachedOps:
-      result.attachedOps[op] = loadSym(c, g, si, item)
   result.typeInst = loadType(c, g, si, t.typeInst)
   for son in items t.types:
     result.sons.add loadType(c, g, si, son)
   loadAstBody(t, n)
-  when false:
-    for gen, id in items t.methods:
-      result.methods.add((gen, loadSym(c, g, si, id)))
 
 proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType =
   if t == nilItemId:

compiler/ic/replayer.nim

@@ -86,6 +86,20 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) =
       else:
         internalAssert g.config, false
 
+proc replayBackendProcs*(g: ModuleGraph; module: int) =
+  for it in mitems(g.packed[module].fromDisk.attachedOps):
+    let key = translateId(it[0], g.packed, module, g.config)
+    let op = it[1]
+    let tmp = translateId(it[2], g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it[2])
+    g.attachedOps[op][key] = LazySym(id: symId, sym: nil)
+
+  for it in mitems(g.packed[module].fromDisk.enumToStringProcs):
+    let key = translateId(it[0], g.packed, module, g.config)
+    let tmp = translateId(it[1], g.packed, module, g.config)
+    let symId = FullId(module: tmp.module, packed: it[1])
+    g.enumToStringProcs[key] = LazySym(id: symId, sym: nil)
+
 proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
   ## We remember the generic instantiations a module performed
   ## in order to to avoid the code bloat that generic code tends
@@ -117,11 +131,7 @@ proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
     let symId = FullId(module: tmp.module, packed: it[2])
     g.methodsPerType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil))
 
-  for it in mitems(g.packed[module].fromDisk.enumToStringProcs):
-    let key = translateId(it[0], g.packed, module, g.config)
-    let tmp = translateId(it[1], g.packed, module, g.config)
-    let symId = FullId(module: tmp.module, packed: it[1])
-    g.enumToStringProcs[key] = LazySym(id: symId, sym: nil)
+  replayBackendProcs(g, module)
 
   for it in mitems(g.packed[module].fromDisk.methods):
     let sym = loadSymFromId(g.config, g.cache, g.packed, module,

compiler/ic/rodfiles.nim

@@ -82,7 +82,6 @@ type
     convertersSection
     methodsSection
     pureEnumsSection
-    macroUsagesSection
     toReplaySection
     topLevelSection
     bodiesSection

compiler/injectdestructors.nim

@@ -964,7 +964,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       result[0] = p(n[0], c, s, mode)
       for i in 1..<n.len:
         result[i] = n[i]
-    of nkGotoState, nkState, nkAsmStmt:
+    of nkGotoState, nkState, nkAsmStmt, nkOpenSymChoice, nkClosedSymChoice:
       result = n
     else:
       internalError(c.graph.config, n.info, "cannot inject destructors to node kind: " & $n.kind)

compiler/int128.nim

@@ -354,18 +354,6 @@ proc low64(a: Int128): uint64 =
   bitconcat(a.udata[1], a.udata[0])
 
 proc `*`*(lhs, rhs: Int128): Int128 =
-  let
-    a = cast[uint64](lhs.udata[0])
-    b = cast[uint64](lhs.udata[1])
-    c = cast[uint64](lhs.udata[2])
-    d = cast[uint64](lhs.udata[3])
-
-    e = cast[uint64](rhs.udata[0])
-    f = cast[uint64](rhs.udata[1])
-    g = cast[uint64](rhs.udata[2])
-    h = cast[uint64](rhs.udata[3])
-
-
   let a32 = cast[uint64](lhs.udata[1])
   let a00 = cast[uint64](lhs.udata[0])
   let b32 = cast[uint64](rhs.udata[1])

compiler/jsgen.nim

@@ -2399,9 +2399,12 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
     else:
       returnStmt = "return $#;$n" % [a.res]
 
-  var transformedBody = transformBody(p.module.graph, p.module.idgen, prc, cache = false)
-  if sfInjectDestructors in prc.flags:
-    transformedBody = injectDestructorCalls(p.module.graph, p.module.idgen, prc, transformedBody)
+  when false:
+    var transformedBody = transformBody(p.module.graph, p.module.idgen, prc, cache = false)
+    if sfInjectDestructors in prc.flags:
+      transformedBody = injectDestructorCalls(p.module.graph, p.module.idgen, prc, transformedBody)
+
+  let transformedBody = getBody(p.module.graph, prc)
 
   p.nested: genStmt(p, transformedBody)
 

compiler/lambdalifting.nim

@@ -12,7 +12,7 @@
 import
   intsets, strutils, options, ast, astalgo, msgs,
   idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos,
-  transf, liftdestructors
+  liftdestructors
 
 discard """
   The basic approach is that captured vars need to be put on the heap and
@@ -436,7 +436,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if innerProc:
       if s.isIterator: c.somethingToDo = true
       if not c.processed.containsOrIncl(s.id):
-        let body = transformBody(c.graph, c.idgen, s, cache = true)
+        let body = getBody(c.graph, s) # transformBody(c.graph, c.idgen, s, useCache)
         detectCapturedVars(body, s, c)
     let ow = s.skipGenericOwner
     if ow == owner:
@@ -732,13 +732,12 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        var body = transformBody(d.graph, d.idgen, s, cache = false)
+        var body = getBody(d.graph, s) #transformBody(d.graph, d.idgen, s, dontUseCache)
         body = liftCapturedVars(body, s, d, c)
-        if c.envVars.getOrDefault(s.id).isNil:
-          s.transformedBody = body
-        else:
-          s.transformedBody = newTree(nkStmtList, rawClosureCreation(s, d, c, n.info), body)
-          finishClosureCreation(s, d, c, n.info, s.transformedBody)
+        if not c.envVars.getOrDefault(s.id).isNil:
+          body = newTree(nkStmtList, rawClosureCreation(s, d, c, n.info), body)
+          finishClosureCreation(s, d, c, n.info, body)
+        setRoutineBody(d.graph, s, body)
         c.inContainer = oldInContainer
 
       if s.typ.callConv == ccClosure:

compiler/main.nim

@@ -17,9 +17,10 @@ import
   condsyms, times,
   sem, idents, passes, extccomp,
   cgen, json, nversion,
-  platform, nimconf, passaux, depends, vm,
+  nimconf, passaux, depends, vm,
   modules,
-  modulegraphs, tables, lineinfos, pathutils, vmprofiler
+  modulegraphs, tables, lineinfos, pathutils, vmprofiler,
+  platform
 
 import ic / [cbackend, integrity, navigator]
 from ic / ic import rodViewer

compiler/modulegraphs.nim

@@ -57,7 +57,7 @@ type
 
     typeInstCache*: Table[ItemId, seq[LazyType]] # A symbol's ItemId.
     procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId.
-    attachedOps*: array[TTypeAttachedOp, Table[ItemId, PSym]] # Type ID, destructors, etc.
+    attachedOps*: array[TTypeAttachedOp, Table[ItemId, LazySym]] # Type ID, destructors, etc.
     methodsPerType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods
     enumToStringProcs*: Table[ItemId, LazySym]
     emittedTypeInfo*: Table[string, FileIndex]
@@ -266,6 +266,13 @@ proc resolveSym(g: ModuleGraph; t: var LazySym): PSym =
     t.sym = result
   assert result != nil
 
+proc resolveAttachedOp(g: ModuleGraph; t: var LazySym): PSym =
+  result = t.sym
+  if result == nil:
+    result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed)
+    t.sym = result
+  assert result != nil
+
 proc resolveInst(g: ModuleGraph; t: var LazyInstantiation): PInstantiation =
   result = t.inst
   if result == nil and isCachedModule(g, t.module):
@@ -292,22 +299,24 @@ iterator procInstCacheItems*(g: ModuleGraph; s: PSym): PInstantiation =
 proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym =
   ## returns the requested attached operation for type `t`. Can return nil
   ## if no such operation exists.
-  result = g.attachedOps[op].getOrDefault(t.itemId)
+  if g.attachedOps[op].contains(t.itemId):
+    result = resolveAttachedOp(g, g.attachedOps[op][t.itemId])
+  else:
+    result = nil
 
 proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
   ## we also need to record this to the packed module.
-  g.attachedOps[op][t.itemId] = value
+  g.attachedOps[op][t.itemId] = LazySym(sym: value)
 
 proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
   ## we also need to record this to the packed module.
-  g.attachedOps[op][t.itemId] = value
-  # XXX Also add to the packed module!
+  g.attachedOps[op][t.itemId] = LazySym(sym: value)
 
 proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
   if g.config.symbolFiles != disabledSf:
     assert module < g.encoders.len
     assert isActive(g.encoders[module])
-    toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].fromDisk)
+    storeAttachedProcDef(t, op, value, g.encoders[module], g.packed[module].fromDisk)
 
 proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
   result = resolveSym(g, g.enumToStringProcs[t.itemId])
@@ -582,3 +591,11 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex;
 
 proc configComplete*(g: ModuleGraph) =
   rememberStartupConfig(g.startupPackedConfig, g.config)
+
+proc setRoutineBody*(g: ModuleGraph; s: PSym; body: PNode) =
+  # XXX Check sanity of IC implementation for this here.
+  s.ast[bodyPos] = body
+
+proc getSemcheckedBody*(g: ModuleGraph; s: PSym): PNode =
+  # XXX IC support
+  result = s.semcheckedBody

compiler/passaux.nim

@@ -10,7 +10,7 @@
 ## implements some little helper passes
 
 import
-  ast, passes, idents, msgs, options, lineinfos
+  ast, passes, msgs, options, lineinfos
 
 from modulegraphs import ModuleGraph, PPassContext