import foo {.all.} reboot (nim-lang#17706)

changelog.md

@@ -307,6 +307,15 @@
 - Added `iterable[T]` type class to match called iterators, which enables writing:
   `template fn(a: iterable)` instead of `template fn(a: untyped)`
 
+- A new import syntax `import foo {.all.}` now allows to import all symbols (public or private)
+  from `foo`. It works in combination with all pre-existing import features.
+  This reduces or eliminates the need for workarounds such as using `include` (which has known issues)
+  when you need a private symbol for testing or making some internal APIs public just because
+  another internal module needs those.
+  It also helps mitigate the lack of cyclic imports in some cases.
+
+- Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields
+  for an object type in the current scope.
 
 ## Compiler changes
 

compiler/ast.nim

@@ -605,6 +605,8 @@ type
 const
   routineKinds* = {skProc, skFunc, skMethod, skIterator,
                    skConverter, skMacro, skTemplate}
+  ExportableSymKinds* = {skVar, skLet, skConst, skType, skEnumField, skStub, skAlias} + routineKinds
+
   tfUnion* = tfNoSideEffect
   tfGcSafe* = tfThread
   tfObjHasKids* = tfEnumHasHoles
@@ -691,7 +693,7 @@ type
     mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2,
     mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples,
     mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf,
-    mSymIsInstantiationOf, mNodeId
+    mSymIsInstantiationOf, mNodeId, mPrivateAccess
 
 
 # things that we can evaluate safely at compile time, even if not asked for it:
@@ -841,6 +843,7 @@ type
     depthLevel*: int
     symbols*: TStrTable
     parent*: PScope
+    allowPrivateAccess*: seq[PSym] #  # enable access to private fields
 
   PScope* = ref TScope
 
@@ -1011,9 +1014,6 @@ const
   NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr,
     tyProc, tyError} # TODO
   PtrLikeKinds*: TTypeKinds = {tyPointer, tyPtr} # for VM
-  ExportableSymKinds* = {skVar, skConst, skProc, skFunc, skMethod, skType,
-    skIterator,
-    skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16,
                                       nfDotSetter, nfDotField,
                                       nfIsRef, nfIsPtr, nfPreventCg, nfLL,

compiler/ic/ic.nim

@@ -36,6 +36,7 @@ type
     bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
     #producedGenerics*: Table[GenericKey, SymId]
     exports*: seq[(LitId, int32)]
+    hidden*: seq[(LitId, int32)]
     reexports*: seq[(LitId, PackedItemId)]
     compilerProcs*: seq[(LitId, int32)]
     converters*, methods*, trmacros*, pureEnums*: seq[int32]
@@ -177,6 +178,10 @@ proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex)
 proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) =
   m.imports.add toLitId(f, c, m)
 
+proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
+  let nameId = getOrIncl(m.sh.strings, s.name.s)
+  m.hidden.add((nameId, s.itemId.item))
+
 proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) =
   let nameId = getOrIncl(m.sh.strings, s.name.s)
   m.exports.add((nameId, s.itemId.item))
@@ -524,7 +529,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadTabSection numbersSection, m.sh.numbers
 
   loadSeqSection exportsSection, m.exports
-
+  loadSeqSection hiddenSection, m.hidden
   loadSeqSection reexportsSection, m.reexports
 
   loadSeqSection compilerProcsSection, m.compilerProcs
@@ -589,7 +594,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeTabSection numbersSection, m.sh.numbers
 
   storeSeqSection exportsSection, m.exports
-
+  storeSeqSection hiddenSection, m.hidden
   storeSeqSection reexportsSection, m.reexports
 
   storeSeqSection compilerProcsSection, m.compilerProcs
@@ -655,7 +660,9 @@ type
     syms: seq[PSym] # indexed by itemId
     types: seq[PType]
     module*: PSym # the one true module symbol.
-    iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too
+    iface, ifaceHidden: Table[PIdent, seq[PackedItemId]]
+      # PackedItemId so that it works with reexported symbols too
+      # ifaceHidden includes private symbols
 
   PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex
 
@@ -882,12 +889,22 @@ proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym
 proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                        fileIdx: FileIndex; m: var LoadedModule) =
   m.iface = initTable[PIdent, seq[PackedItemId]]()
-  for e in m.fromDisk.exports:
+  m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]()
+  template impl(iface, e) =
     let nameLit = e[0]
-    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1]))
-  for re in m.fromDisk.reexports:
-    let nameLit = re[0]
-    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1])
+    let e2 =
+      when e[1] is PackedItemId: e[1]
+      else: PackedItemId(module: LitId(0), item: e[1])
+    iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(e2)
+
+  for e in m.fromDisk.exports:
+    m.iface.impl(e)
+    m.ifaceHidden.impl(e)
+  for e in m.fromDisk.reexports:
+    m.iface.impl(e)
+    m.ifaceHidden.impl(e)
+  for e in m.fromDisk.hidden:
+    m.ifaceHidden.impl(e)
 
   let filename = AbsoluteFile toFullPath(conf, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
@@ -1053,24 +1070,29 @@ type
     values: seq[PackedItemId]
     i, module: int
 
+template interfSelect(a: LoadedModule, importHidden: bool): auto =
+  var ret = a.iface.addr
+  if importHidden: ret = a.ifaceHidden.addr
+  ret[]
+
 proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache;
                   g: var PackedModuleGraph; module: FileIndex;
-                  name: PIdent): PSym =
+                  name: PIdent, importHidden: bool): PSym =
   it.decoder = PackedDecoder(
     lastModule: int32(-1),
     lastLit: LitId(0),
     lastFile: FileIndex(-1),
     config: config,
     cache: cache)
-  it.values = g[int module].iface.getOrDefault(name)
+  it.values = g[int module].interfSelect(importHidden).getOrDefault(name)
   it.i = 0
   it.module = int(module)
   if it.i < it.values.len:
     result = loadSym(it.decoder, g, int(module), it.values[it.i])
     inc it.i
 
 proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
-                         g: var PackedModuleGraph; module: FileIndex): PSym =
+                         g: var PackedModuleGraph; module: FileIndex, importHidden: bool): PSym =
   it.decoder = PackedDecoder(
     lastModule: int32(-1),
     lastLit: LitId(0),
@@ -1079,7 +1101,7 @@ proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
     cache: cache)
   it.values = @[]
   it.module = int(module)
-  for v in g[int module].iface.values:
+  for v in g[int module].interfSelect(importHidden).values:
     it.values.add v
   it.i = 0
   if it.i < it.values.len:
@@ -1093,19 +1115,19 @@ proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym =
 
 iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache;
                            g: var PackedModuleGraph; module: FileIndex;
-                           name: PIdent): PSym =
+                           name: PIdent, importHidden: bool): PSym =
   setupDecoder()
-  let values = g[int module].iface.getOrDefault(name)
+  let values = g[int module].interfSelect(importHidden).getOrDefault(name)
   for pid in values:
     let s = loadSym(decoder, g, int(module), pid)
     assert s != nil
     yield s
 
 proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
                       g: var PackedModuleGraph; module: FileIndex;
-                      name: PIdent): PSym =
+                      name: PIdent, importHidden: bool): PSym =
   setupDecoder()
-  let values = g[int module].iface.getOrDefault(name)
+  let values = g[int module].interfSelect(importHidden).getOrDefault(name)
   result = loadSym(decoder, g, int(module), values[0])
 
 proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
@@ -1140,6 +1162,10 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
       echo "  ", m.sh.strings[ex[0]]
     #  reexports*: seq[(LitId, PackedItemId)]
 
+    echo "hidden: " & $m.hidden.len
+    for ex in m.hidden:
+      echo "  ", m.sh.strings[ex[0]], " local ID: ", ex[1]
+
   echo "all symbols"
   for i in 0..high(m.sh.syms):
     if m.sh.syms[i].name != LitId(0):

compiler/ic/rodfiles.nim

@@ -18,6 +18,7 @@ type
     depsSection
     numbersSection
     exportsSection
+    hiddenSection
     reexportsSection
     compilerProcsSection
     trmacrosSection

compiler/importer.nim

@@ -12,7 +12,7 @@
 import
   intsets, ast, astalgo, msgs, options, idents, lookups,
   semdata, modulepaths, sigmatch, lineinfos, sets,
-  modulegraphs
+  modulegraphs, wordrecg
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
@@ -107,7 +107,25 @@ proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) =
   if s.owner != origin:
     c.exportIndirections.incl((origin.id, s.id))
 
+proc splitPragmas(c: PContext, n: PNode): (PNode, seq[TSpecialWord]) =
+  template bail = globalError(c.config, n.info, "invalid pragma")
+  if n.kind == nkPragmaExpr:
+    if n.len == 2 and n[1].kind == nkPragma:
+      result[0] = n[0]
+      for ni in n[1]:
+        if ni.kind == nkIdent: result[1].add whichKeyword(ni.ident)
+        else: bail()
+    else: bail()
+  else:
+    result[0] = n
+    if result[0].safeLen > 0:
+      (result[0][^1], result[1]) = splitPragmas(c, result[0][^1])
+
 proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
+  let (n, kws) = splitPragmas(c, n)
+  if kws.len > 0:
+    globalError(c.config, n.info, "unexpected pragma")
+
   let ident = lookups.considerQuotedIdent(c, n)
   let s = someSym(c.graph, fromMod, ident)
   if s == nil:
@@ -204,18 +222,43 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im
     for i in 0..n.safeLen-1:
       importForwarded(c, n[i], exceptSet, fromMod, importSet)
 
-proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
+proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym =
   result = realModule
   c.unusedImports.add((realModule, n.info))
+  template createModuleAliasImpl(ident): untyped =
+    createModuleAlias(realModule, nextSymId c.idgen, ident, realModule.info, c.config.options)
   if n.kind != nkImportAs: discard
   elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
   elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAlias(realModule, nextSymId c.idgen, n[1].ident, realModule.info,
-                               c.config.options)
+    result = createModuleAliasImpl(n[1].ident)
+  if importHidden:
+    if result == realModule: # avoids modifying `realModule`, see D20201209T194412.
+      result = createModuleAliasImpl(realModule.name)
+    result.options.incl optImportHidden
+
+proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] =
+  var ret: typeof(result)
+  proc processPragma(n2: PNode): PNode =
+    let (result2, kws) = splitPragmas(c, n2)
+    result = result2
+    for ai in kws:
+      case ai
+      of wImportHidden: ret.importHidden = true
+      else: globalError(c.config, n.info, "invalid pragma, expected: " & ${wImportHidden})
+
+  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
+    ret.node = newNodeI(nkImportAs, n.info)
+    ret.node.add n[1].processPragma
+    ret.node.add n[2]
+  else:
+    ret.node = n.processPragma
+  return ret
 
-proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
+proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym =
+  let transf = transformImportAs(c, n)
+  n = transf.node
   let f = checkModuleName(c.config, n)
   if f != InvalidFileIdx:
     addImportFileDep(c, f)
@@ -232,7 +275,7 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
       c.recursiveDep = err
 
     discard pushOptionEntry(c)
-    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
+    result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f), transf.importHidden)
     popOptionEntry(c)
 
     #echo "set back to ", L
@@ -252,16 +295,8 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
     importStmtResult.add newSymNode(result, n.info)
     #newStrNode(toFullPath(c.config, f), n.info)
 
-proc transformImportAs(c: PContext; n: PNode): PNode =
-  if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
-    result = newNodeI(nkImportAs, n.info)
-    result.add n[1]
-    result.add n[2]
-  else:
-    result = n
-
 proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
-  let it = transformImportAs(c, it)
+  var it = it
   let m = myImportModule(c, it, importStmtResult)
   if m != nil:
     # ``addDecl`` needs to be done before ``importAllSymbols``!
@@ -296,7 +331,6 @@ proc evalImport*(c: PContext, n: PNode): PNode =
 proc evalFrom*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)
@@ -311,7 +345,6 @@ proc evalFrom*(c: PContext, n: PNode): PNode =
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n[0] = transformImportAs(c, n[0])
   var m = myImportModule(c, n[0], result)
   if m != nil:
     n[0] = newSymNode(m)

compiler/lookups.nim

@@ -75,7 +75,7 @@ proc closeScope*(c: PContext) =
   ensureNoMissingOrUnusedSymbols(c, c.currentScope)
   rawCloseScope(c)
 
-iterator allScopes(scope: PScope): PScope =
+iterator allScopes*(scope: PScope): PScope =
   var current = scope
   while current != nil:
     yield current
@@ -311,11 +311,17 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) =
   if conflict != nil:
     wrongRedefinition(c, sym.info, sym.name.s, conflict.info)
 
-proc addInterfaceDeclAux(c: PContext, sym: PSym) =
-  if sfExported in sym.flags:
+from ic / ic import addHidden
+
+proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) =
+  if sfExported in sym.flags or forceExport:
     # add to interface:
     if c.module != nil: exportSym(c, sym)
     else: internalError(c.config, sym.info, "addInterfaceDeclAux")
+  elif sym.kind in ExportableSymKinds and c.module != nil and isTopLevelInsideDeclaration(c, sym):
+    strTableAdd(semtabAll(c.graph, c.module), sym)
+    if c.config.symbolFiles != disabledSf:
+      addHidden(c.encoder, c.packedRepr, sym)
 
 proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) =
   addDeclAt(c, scope, sym)

compiler/magicsys.nim

@@ -50,6 +50,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
   result = g.sysTypes[kind]
   if result == nil:
     case kind
+    of tyVoid: result = sysTypeFromName("void")
     of tyInt: result = sysTypeFromName("int")
     of tyInt8: result = sysTypeFromName("int8")
     of tyInt16: result = sysTypeFromName("int16")