[superseded] import foo {.all.}

changelog.md

@@ -66,6 +66,12 @@
 
 - nil dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time.
 
+- A new import syntax `import {.all} foo` now allows to import all symbols (public or private)
+  from `foo`. It works in combination with all pre-existing import features. This
+  helps avoid using workarounds such as using `include` when you need a private symbol for
+  testing (which has known issues) or making some internal APIs public just because another internal
+  module needs those. It also helps mitigate the lack of cyclic imports in some cases.
+
 
 ## Compiler changes
 

compiler/ast.nim

@@ -843,7 +843,8 @@ type
       # No need, just leave it as skModule but set the owner accordingly and
       # check for the owner when touching 'usedGenerics'.
       usedGenerics*: seq[PInstantiation]
-      tab*: TStrTable         # interface table for modules
+      tab*: TStrTable            # interface table for modules
+      tabAll*: TStrTable         # interface table for modules (all top-level)
     of skLet, skVar, skField, skForVar:
       guard*: PSym
       bitsize*: int
@@ -1437,6 +1438,7 @@ proc copySym*(s: PSym; id: ItemId): PSym =
   result.magic = s.magic
   if s.kind == skModule:
     copyStrTable(result.tab, s.tab)
+    copyStrTable(result.tabAll, s.tabAll)
   result.options = s.options
   result.position = s.position
   result.loc = s.loc
@@ -1455,6 +1457,7 @@ proc createModuleAlias*(s: PSym, id: ItemId, newIdent: PIdent, info: TLineInfo;
   #result.id = s.id # XXX figure out what to do with the ID.
   result.flags = s.flags
   system.shallowCopy(result.tab, s.tab)
+  system.shallowCopy(result.tabAll, s.tabAll)
   result.options = s.options
   result.position = s.position
   result.loc = s.loc
@@ -1954,6 +1957,17 @@ proc canRaise*(fn: PNode): bool =
       (fn.typ.n[0][exceptionEffects] != nil and
       fn.typ.n[0][exceptionEffects].safeLen > 0))
 
+template addPublicSymbol*(module: PSym, s: PSym) =
+  strTableAdd(module.tab, s)
+  strTableAdd(module.tabAll, s)
+
+template tabOpt*(m: PSym): untyped =
+  # avoids doing a copy
+  let tab2 =
+    if optImportAll in m.options: m.tabAll.addr
+    else: m.tab.addr
+  tab2[]
+
 proc toHumanStrImpl[T](kind: T, num: static int): string =
   result = $kind
   result = result[num..^1]

compiler/importer.nim

@@ -11,7 +11,7 @@
 
 import
   intsets, ast, astalgo, msgs, options, idents, lookups,
-  semdata, modulepaths, sigmatch, lineinfos, sets
+  semdata, modulepaths, sigmatch, lineinfos, sets, wordrecg
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
@@ -72,9 +72,20 @@ proc rawImportSymbol(c: PContext, s, origin: PSym) =
   if s.owner != origin:
     c.exportIndirections.incl((origin.id, s.id))
 
+template getTab(c: PContext, fromMod: PSym): untyped =
+  # avoids doing a copy
+  let tab2 =
+    if fromMod in c.friendModulesImportAll:
+      # so that `import foo {.all/}` also works with `import foo` without `as`
+      # another option is to force `createModuleAlias` to remove this branch
+      fromMod.tabAll.addr
+    else:
+      fromMod.tabOpt.addr
+  tab2[]
+
 proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   let ident = lookups.considerQuotedIdent(c, n)
-  let s = strTableGet(fromMod.tab, ident)
+  let s = strTableGet(c.getTab(fromMod), ident)
   if s == nil:
     errorUndeclaredIdentifier(c, n.info, ident.s)
   else:
@@ -85,27 +96,27 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
     if multiImport:
       # for a overloadable syms add all overloaded routines
       var it: TIdentIter
-      var e = initIdentIter(it, fromMod.tab, s.name)
+      var e = initIdentIter(it, c.getTab(fromMod), s.name)
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
         if s.kind in ExportableSymKinds:
           rawImportSymbol(c, e, fromMod)
-        e = nextIdentIter(it, fromMod.tab)
+        e = nextIdentIter(it, c.getTab(fromMod))
     else:
       rawImportSymbol(c, s, fromMod)
     suggestSym(c.config, n.info, s, c.graph.usageSym, false)
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
   var i: TTabIter
-  var s = initTabIter(i, fromMod.tab)
+  var s = initTabIter(i, c.getTab(fromMod))
   while s != nil:
     if s.kind != skModule:
       if s.kind != skEnumField:
         if s.kind notin ExportableSymKinds:
           internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
         if exceptSet.isNil or s.name.id notin exceptSet:
           rawImportSymbol(c, s, fromMod)
-    s = nextIter(i, fromMod.tab)
+    s = nextIter(i, c.getTab(fromMod))
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
   var exceptSet: IntSet
@@ -128,7 +139,12 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym) =
     for i in 0..n.safeLen-1:
       importForwarded(c, n[i], exceptSet, fromMod)
 
-proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
+type
+  ImportFlag = enum
+    ifImportAll
+  ImportFlags = set[ImportFlag]
+
+proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importFlags: ImportFlags): PSym =
   result = realModule
   c.unusedImports.add((realModule, n.info))
   if n.kind != nkImportAs: discard
@@ -138,8 +154,38 @@ proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
     # some misguided guy will write 'import abc.foo as foo' ...
     result = createModuleAlias(realModule, nextId c.idgen, n[1].ident, realModule.info,
                                c.config.options)
+  if ifImportAll in importFlags:
+    if result == realModule:
+      result = createModuleAlias(realModule, nextId c.idgen, realModule.name, realModule.info,
+                               c.config.options)
+    result.options.incl optImportAll
+    c.friendModulesImportAll.add realModule # `realModule` needed, not `result`
+
+proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importFlags: ImportFlags] =
+  var ret: typeof(result)
+  proc processPragma(n2: PNode): PNode =
+    if n2.kind == nkPragmaExpr:
+      if n2.len == 2 and n2[1].kind == nkPragma and n2[1].len == 1 and n2[1][0].kind == nkIdent and whichKeyword(n2[1][0].ident) == wImportAll: discard
+      else:
+        globalError(c.config, n.info, "invalid import pragma, expected: " & $wImportAll)
+      ret.importFlags.incl ifImportAll
+      result = n2[0]
+    else:
+      result = n2
+      if result.safeLen > 0:
+        result[^1] = processPragma(result[^1])
 
-proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
+  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
+    ret.node = newNodeI(nkImportAs, n.info)
+    ret.node.add n[1].processPragma
+    ret.node.add n[2]
+  else:
+    ret.node = n.processPragma
+  return ret
+
+proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym =
+  var importFlags: ImportFlags
+  (n,importFlags) = transformImportAs(c, n)
   let f = checkModuleName(c.config, n)
   if f != InvalidFileIdx:
     let L = c.graph.importStack.len
@@ -155,7 +201,7 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
       c.recursiveDep = err
 
     discard pushOptionEntry(c)
-    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
+    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f), importFlags)
     popOptionEntry(c)
 
     #echo "set back to ", L
@@ -175,16 +221,8 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
 
-proc transformImportAs(c: PContext; n: PNode): PNode =
-  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
-    result = newNodeI(nkImportAs, n.info)
-    result.add n[1]
-    result.add n[2]
-  else:
-    result = n
-
 proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
-  let it = transformImportAs(c, it)
+  var it = it
   let m = myImportModule(c, it, importStmtResult)
   if m != nil:
     # ``addDecl`` needs to be done before ``importAllSymbols``!
@@ -219,7 +257,6 @@ proc evalImport*(c: PContext, n: PNode): PNode =
 proc evalFrom*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)
@@ -231,7 +268,6 @@ proc evalFrom*(c: PContext, n: PNode): PNode =
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)

compiler/lookups.nim

@@ -361,7 +361,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
         if m == c.module:
           result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config)
         else:
-          result = strTableGet(m.tab, ident).skipAlias(n, c.config)
+          result = strTableGet(m.tabOpt, ident).skipAlias(n, c.config)
         if result == nil and checkUndeclared in flags:
           fixSpelling(n[1], ident, searchInScopes)
           errorUndeclaredIdentifier(c, n[1].info, ident.s)
@@ -410,7 +410,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
                                  ident).skipAlias(n, c.config)
           o.mode = oimSelfModule
         else:
-          result = initIdentIter(o.it, o.m.tab, ident).skipAlias(n, c.config)
+          result = initIdentIter(o.it, o.m.tabOpt, ident).skipAlias(n, c.config)
       else:
         noidentError(c.config, n[1], n)
         result = errorSym(c, n[1])
@@ -452,7 +452,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   of oimSelfModule:
     result = nextIdentIter(o.it, c.topLevelScope.symbols).skipAlias(n, c.config)
   of oimOtherModule:
-    result = nextIdentIter(o.it, o.m.tab).skipAlias(n, c.config)
+    result = nextIdentIter(o.it, o.m.tabOpt).skipAlias(n, c.config)
   of oimSymChoice:
     if o.symChoiceIndex < n.len:
       result = n[o.symChoiceIndex].sym

compiler/modulegraphs.nim

@@ -137,31 +137,43 @@ proc hash*(u: SigHash): Hash =
 
 proc hash*(x: FileIndex): Hash {.borrow.}
 
+template getC(): untyped =
+  when compiles(c.c.graph): c.c
+  else: c
+
+template onDefAux(info: TLineInfo; s0: PSym, c0: untyped, isFwd: bool) =
+  if s0.kind in ExportableSymKinds:
+    let c = c0 # in case c0 is an expression
+    var top = true
+    case s0.kind
+    of skProc, skMacro:
+      # unfortunately, can't use `c.isTopLevel` because the scope isn't closed yet
+      top = c.currentScope.depthLevel <= 3
+    else: top = c.currentScope.depthLevel <= 2
+    if top:
+      let loc = toFileLineCol(c.config, info)
+      if c.module != nil: strTableAdd(c.module.tabAll, s0)
+
 when defined(nimfind):
   template onUse*(info: TLineInfo; s: PSym) =
-    when compiles(c.c.graph):
-      if c.c.graph.onUsage != nil: c.c.graph.onUsage(c.c.graph, s, info)
-    else:
-      if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info)
+    let c2 = getC()
+    if c2.graph.onUsage != nil: c2.graph.onUsage(c2.graph, s, info)
 
   template onDef*(info: TLineInfo; s: PSym) =
-    when compiles(c.c.graph):
-      if c.c.graph.onDefinition != nil: c.c.graph.onDefinition(c.c.graph, s, info)
-    else:
-      if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info)
+    let c2 = getC()
+    onDefAux(info, s, c2, false)
+    if c2.graph.onDefinition != nil: c2.graph.onDefinition(c2.graph, s, info)
 
   template onDefResolveForward*(info: TLineInfo; s: PSym) =
-    when compiles(c.c.graph):
-      if c.c.graph.onDefinitionResolveForward != nil:
-        c.c.graph.onDefinitionResolveForward(c.c.graph, s, info)
-    else:
-      if c.graph.onDefinitionResolveForward != nil:
-        c.graph.onDefinitionResolveForward(c.graph, s, info)
+    let c2 = getC()
+    onDefAux(info, s, c2, true)
+    if c2.graph.onDefinitionResolveForward != nil:
+      c2.graph.onDefinitionResolveForward(c2.graph, s, info)
 
 else:
   template onUse*(info: TLineInfo; s: PSym) = discard
-  template onDef*(info: TLineInfo; s: PSym) = discard
-  template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
+  template onDef*(info: TLineInfo; s: PSym) = onDefAux(info, s, getC(), false)
+  template onDefResolveForward*(info: TLineInfo; s: PSym) = onDefAux(info, s, getC(), true)
 
 proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()

compiler/modules.nim

@@ -64,7 +64,8 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
   graph.modules[result.position] = result
 
   initStrTable(result.tab)
-  strTableAdd(result.tab, result) # a module knows itself
+  initStrTable(result.tabAll)
+  result.addPublicSymbol result # a module knows itself
   strTableAdd(packSym.tab, result)
 
 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
@@ -103,6 +104,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
     result.flags.excl sfDirty
     # reset module fields:
     initStrTable(result.tab)
+    initStrTable(result.tabAll)
     result.ast = nil
     processModuleAux()
     graph.markClientsDirty(fileIdx)

compiler/options.nim

@@ -40,9 +40,9 @@ type                          # please make sure we have under 32 options
     optTrMacros,              # en/disable pattern matching
     optMemTracker,
     optNilSeqs,
-    optSinkInference          # 'sink T' inference
-    optCursorInference
-
+    optSinkInference,         # 'sink T' inference
+    optCursorInference,
+    optImportAll,
 
   TOptions* = set[TOption]
   TGlobalOption* = enum       # **keep binary compatible**
@@ -176,7 +176,7 @@ type
       ## Note: this feature can't be localized with {.push.}
     vmopsDanger,
     strictFuncs,
-    views
+    views,
 
   LegacyFeature* = enum
     allowSemcheckedAstModification,

compiler/semdata.nim

@@ -83,6 +83,7 @@ type
     friendModules*: seq[PSym]  # friend modules; may access private data;
                                # this is used so that generic instantiations
                                # can access private object fields
+    friendModulesImportAll*: seq[PSym] # enable access to private fields
     instCounter*: int          # to prevent endless instantiations
     templInstCounter*: ref int # gives every template instantiation a unique id