support foreign TLib references

A symbol can now reference the `TLib` belonging to a module that is not
the one the symbol is attached to. This works around an issue with
aliased structural types.

Also, use the ID provided by the module's ID generator instead of the
`skModule`s position. While both represent the same thing (a
``FileIndex``), querying the ID generator is more consistent with how it
works elsewhere.

compiler/ast/ast.nim

@@ -29,9 +29,6 @@ import
   std/[
     strutils,
     tables # For symbol table mapping
-  ],
-  experimental/[
-    dod_helpers
   ]
 
 export ast_types, ast_idgen, ast_query, int128
@@ -327,7 +324,7 @@ proc assignType*(dest, src: PType) =
   if src.sym != nil:
     if dest.sym != nil:
       dest.sym.flags.incl src.sym.flags-{sfExported}
-      if dest.sym.annex.isNone: dest.sym.annex = src.sym.annex
+      if dest.sym.annex.isNil: dest.sym.annex = src.sym.annex
       dest.sym.extFlags.incl src.sym.extFlags
       if dest.sym.extname.len == 0:
         dest.sym.extname = src.sym.extname

compiler/ast/ast_types.nim

@@ -1,7 +1,6 @@
 import compiler/ast/lineinfos
 import compiler/utils/ropes
 import std/[hashes]
-import experimental/dod_helpers
 
 from compiler/ast/idents import PIdent, TIdent
 
@@ -45,16 +44,11 @@ type
 
 type
   NodeId* = distinct int32
-  LibId* = distinct uint32
 
 proc `==`*(a, b: NodeId): bool {.borrow.}
 proc hash*(a: NodeId): Hash {.borrow.}
 proc `$`*(a: NodeId): string {.borrow.}
 
-template indexLike*(_: typedesc[LibId]) =
-  # makes ``LibId`` available to use with ``opt``
-  discard
-
 type
   TNodeKind* = enum
     ## order is important, because ranges are used to check whether a node
@@ -1621,6 +1615,20 @@ type
     name*: Rope
     path*: PNode              ## can be a string literal!
 
+  LibId* = object
+    ## Identifies a ``TLib`` instance. The default value means 'none'.
+    # XXX: ideally, a ``LibId`` would be a single 32-bit index into the
+    #      surrounding module, but this is not possible at the moment, because
+    #      of how aliased structural types work.
+    #
+    #        type A {.header: ... .} = int # declared in module 'A'
+    #        type B = A                    # declared in module 'B'
+    #
+    #      Here, 'B' is not a ``tyAlias`` type, but rather a ``tyInt``, with
+    #      the symbol information from 'A' (including the ``LibId``) copied
+    #      over.
+    module*: int32   ## the ID of the module the lib object is part
+    index*: uint32   ## 1-based index. Zero means 'none'
 
   CompilesId* = int ## id that is used for the caching logic within
                     ## ``system.compiles``. See the seminst module.
@@ -1695,7 +1703,7 @@ type
                               ## generation
     locId*: uint32            ## associates the symbol with a loc in the C code
                               ## generator. 0 means unset.
-    annex*: opt(LibId)        ## additional fields (seldom used, so we use a
+    annex*: LibId             ## additional fields (seldom used, so we use a
                               ## reference to another object to save space)
     constraint*: PNode        ## additional constraints like 'lit|result'; also
                               ## misused for the codegenDecl pragma in the hope
@@ -1897,3 +1905,5 @@ proc `comment=`*(n: PNode, a: string) =
     gconfig.comments.del(n.id)
 
 proc setUseIc*(useIc: bool) = gconfig.useIc = useIc
+
+func isNil*(id: LibId): bool = id.index == 0

compiler/backend/backends.nim

@@ -36,9 +36,6 @@ import
   compiler/utils/[
     containers,
     idioms
-  ],
-  experimental/[
-    dod_helpers
   ]
 
 type
@@ -568,7 +565,7 @@ proc preprocessDynlib(graph: ModuleGraph, idgen: IdGenerator,
   #       be removed once handling of dynlib procedures and globals is fully
   #       implemented in the ``process`` iterator
   if exfDynamicLib in sym.extFlags:
-    let lib = addr graph.libs[moduleId(sym)][sym.annex[]]
+    let lib = addr graph.getLib(sym.annex)
     if lib.path.kind in nkStrKinds:
       # it's a string, no need to transform nor scan it
       discard

compiler/backend/ccgtypes.nim

@@ -11,7 +11,7 @@
 
 # ------------------------- Name Mangling --------------------------------
 
-import compiler/sem/sighashes, compiler/modules/modulegraphs
+import compiler/sem/sighashes
 
 proc isKeyword(w: PIdent): bool =
   # Nim and C++ share some keywords

compiler/backend/cgen.nim

@@ -35,13 +35,13 @@ import
   ],
   compiler/modules/[
     magicsys,
+    modulegraphs
   ],
   compiler/front/[
     options,
     msgs
   ],
   compiler/utils/[
-    containers,
     platform,
     nversion,
     bitsets,
@@ -59,8 +59,7 @@ import
     ccgutils,
     cgendata
   ],
-  experimental/[
-    dod_helpers
+  compiler/plugins/[
   ]
 
 # xxx: reports are a code smell meaning data types are misplaced...
@@ -147,7 +146,7 @@ proc isSimpleConst(typ: PType): bool =
 
 proc useHeader(m: BModule, sym: PSym) =
   if exfHeader in sym.extFlags:
-    let str = getStr(m.g.graph.libs[sym.itemId.module][sym.annex[]].path)
+    let str = getStr(m.g.graph.getLib(sym.annex).path)
     m.includeHeader(str)
 
 proc cgsym(m: BModule, name: string): Rope
@@ -686,7 +685,7 @@ proc mangleDynLibProc(sym: PSym): Rope =
     result = rope(strutils.`%`("Dl_$1_", $sym.id))
 
 proc symInDynamicLib*(m: BModule, sym: PSym) =
-  var lib = addr m.g.graph.libs[sym.itemId.module][sym.annex[]]
+  var lib = addr m.g.graph.getLib(sym.annex)
   let isCall = isGetProcAddr(lib[])
   let extname = sym.extname
   if not isCall: loadDynamicLib(m, lib[])
@@ -725,7 +724,7 @@ proc symInDynamicLib*(m: BModule, sym: PSym) =
   m.s[cfsVars].addf("$2 $1;$n", [tmp, getTypeDesc(m, sym.typ, skVar)])
 
 proc varInDynamicLib(m: BModule, sym: PSym) =
-  var lib = addr m.g.graph.libs[sym.itemId.module][sym.annex[]]
+  var lib = addr m.g.graph.getLib(sym.annex)
   let extname = sym.extname
   loadDynamicLib(m, lib[])
   let tmp = mangleDynLibProc(sym)

compiler/ic/ic.nim

@@ -25,7 +25,6 @@ import
     options
   ],
   compiler/utils/[
-    containers,
     ropes,
     pathutils
   ]
@@ -1097,10 +1096,10 @@ proc loadSymFromId*(config: ConfigRef, cache: IdentCache;
     result = loadSym(decoder, g, module, id)
 
 proc loadLibs*(config: ConfigRef, cache: IdentCache,
-               g: var PackedModuleGraph, module: int): Store[LibId, TLib] =
+               g: var PackedModuleGraph, module: int): seq[TLib] =
   setupDecoder()
   for it in g[module].fromDisk.libs.items:
-    discard result.add(loadLib(decoder, g, module, it))
+    result.add(loadLib(decoder, g, module, it))
 
 proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; config: ConfigRef): ItemId =
   if id.module == LitId(0):

compiler/ic/packed_ast.nim

@@ -13,7 +13,6 @@
 ## it is superior.
 
 import std/[hashes, tables, strtabs]
-import experimental/[dod_helpers]
 import compiler/ic/bitabs
 import compiler/ast/ast, compiler/front/options
 
@@ -62,7 +61,7 @@ type
     offset*: int
     externalName*: LitId # instead of TLoc
     extFlags*: ExternalFlags
-    annex*: opt(LibId)
+    annex*: LibId
     constraint*: NodeId
 
   PackedType* = object

compiler/modules/modulegraphs.nim

@@ -99,7 +99,7 @@ type
     enumToStringProcs*: Table[ItemId, LazySym]
     emittedTypeInfo*: Table[string, FileIndex]
 
-    libs*: seq[Store[LibId, TLib]] ## indexed by module position
+    libs*: seq[seq[TLib]] ## indexed by ``LibId``
 
     startupPackedConfig*: PackedConfig
     packageSyms*: TStrTable
@@ -380,9 +380,23 @@ proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
 proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) =
   g.enumToStringProcs[t.itemId] = LazySym(sym: value)
 
-proc storeLib*(g: ModuleGraph, module: int, lib: TLib) =
+func getLib*(g: ModuleGraph, id: LibId): var TLib =
+  assert not isNil(id), "id is 'none'"
+  result = g.libs[id.module][id.index - 1]
+
+proc addLib*(g: ModuleGraph, module: int, lib: sink TLib): LibId =
+  ## Registers (adds) `lib` with the given module and returns the ID through
+  ## which the instance can be accessed from now on.
+  g.libs[module].add lib
+  # the index is 1-based, so we can directly use the length
+  result = LibId(module: module.int32, index: g.libs[module].len.uint32)
+
+proc storeLibs*(g: ModuleGraph, module: int) =
+  ## Writes the ``TLib`` instances associated with `module` to the module's
+  ## packed representation. Only relevant for IC.
   if g.config.symbolFiles != disabledSf:
-    storeLib(g.encoders[module], g.packed[module].fromDisk, lib)
+    for lib in g.libs[module].items:
+      storeLib(g.encoders[module], g.packed[module].fromDisk, lib)
 
 iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) =
   if g.methodsPerType.contains(t.itemId):

compiler/sem/pragmas.nim

@@ -35,7 +35,6 @@ import
     options
   ],
   compiler/utils/[
-    containers,
     pathutils,
     debugutils,
     idioms
@@ -46,9 +45,6 @@ import
   ],
   compiler/backend/[
     extccomp
-  ],
-  experimental/[
-    dod_helpers
   ]
 
 # xxx: reports are a code smell meaning data types are misplaced
@@ -359,7 +355,7 @@ proc processCallConv(c: PContext, n: PNode): PNode =
     result = c.config.newError(n, PAstDiag(kind: adSemCallconvExpected))
 
 proc getLib(c: PContext, kind: TLibKind, path: PNode): LibId =
-  for id, it in c.libs.pairs:
+  for id, it in c.libs:
     if it.kind == kind and trees.exprStructuralEquivalent(it.path, path):
       return id
 
@@ -368,7 +364,7 @@ proc getLib(c: PContext, kind: TLibKind, path: PNode): LibId =
   if path.kind in {nkStrLit..nkTripleStrLit}:
     lib.isOverriden = options.isDynlibOverride(c.config, path.strVal)
 
-  result = c.libs.add(lib)
+  result = c.addLib(lib)
 
 proc expectDynlibNode(c: PContext, n: PNode): PNode =
   ## `n` must be a callable, this will produce the ast for the callable or
@@ -395,8 +391,8 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym): PNode =
       result = libNode
     else:
       let lib = getLib(c, libDynamic, libNode)
-      if not c.libs[lib].isOverriden:
-        c.optionStack[^1].dynlib = someOpt(lib)
+      if not c[lib].isOverriden:
+        c.optionStack[^1].dynlib = lib
   else:
     if n.kind in nkPragmaCallKinds:
       let libNode = expectDynlibNode(c, n)
@@ -405,7 +401,7 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym): PNode =
         result = libNode
       else:
         var lib = getLib(c, libDynamic, libNode)
-        if not c.libs[lib].isOverriden:
+        if not c[lib].isOverriden:
           addToLib(lib, sym)
           incl(sym.extFlags, exfDynamicLib)
     else:
@@ -1857,10 +1853,10 @@ proc inheritDynlib*(c: PContext, sym: PSym) =
   ## applicable. The dynlib pragma can be applied if the symbol is marked as
   ## imported, but no header nor dynlib are specified.
   let lib = c.optionStack[^1].dynlib
-  if lib.isSome and sfImportc in sym.flags and
+  if not lib.isNil and sfImportc in sym.flags and
      {exfDynamicLib, exfHeader} * sym.extFlags == {}:
     incl(sym.extFlags, exfDynamicLib)
-    addToLib(lib[], sym)
+    addToLib(lib, sym)
     if sym.extname == "":
       # XXX: this looks like a unnecessary defensive check. If the symbol is
       #      marked as imported, it already has an external name set

compiler/sem/sem.nim

@@ -46,7 +46,6 @@ import
     msgs
   ],
   compiler/utils/[
-    containers,
     ropes,
     platform,
     nversion,
@@ -939,10 +938,7 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
   var c = PContext(context)
   if c.config.cmd == cmdIdeTools and not c.suggestionsMade:
     suggestSentinel(c)
-
-  for it in c.libs.items:
-    storeLib(graph, c.module.position, it)
-
+  storeLibs(graph, c.idgen.module)
   closeScope(c)         # close module's scope
   rawCloseScope(c)      # imported symbols; don't check for unused ones!
   reportUnusedModules(c)

compiler/sem/semdata.nim

@@ -36,12 +36,6 @@ import
   ],
   compiler/ic/[
     ic
-  ],
-  compiler/utils/[
-    containers
-  ],
-  experimental/[
-    dod_helpers
   ]
 
 from compiler/ast/reports_sem import reportAst,
@@ -57,7 +51,7 @@ type
   TOptionEntry* = object      ## entries to put on a stack for pragma parsing
     options*: TOptions
     defaultCC*: TCallingConvention
-    dynlib*: opt(LibId)
+    dynlib*: LibId
     notes*: ReportKinds
     features*: set[Feature]
     otherPragmas*: PNode      ## every pragma can be pushed
@@ -799,7 +793,7 @@ proc newOptionEntry*(conf: ConfigRef): POptionEntry =
   new(result)
   result.options = conf.options
   result.defaultCC = ccNimCall
-  result.dynlib = noneOpt(LibId)
+  result.dynlib = LibId()
   result.notes = conf.notes
   result.warningAsErrors = conf.warningAsErrors
 
@@ -897,16 +891,26 @@ proc reexportSym*(c: PContext; s: PSym) =
   if c.config.symbolFiles != disabledSf:
     addReexport(c.encoder, c.packedRepr, s)
 
-template libs*(c: PContext): Store[LibId, TLib] =
-  c.graph.libs[c.module.position]
-
 proc initLib*(kind: TLibKind): TLib =
   result = TLib(kind: kind)
 
 proc addToLib*(lib: LibId, sym: PSym) =
   #if sym.annex != nil and not isGenericRoutine(sym):
   #  LocalError(sym.info, errInvalidPragma)
-  sym.annex = someOpt(lib)
+  assert not isNil(lib)
+  sym.annex = lib
+
+proc addLib*(c: PContext, lib: sink TLib): LibId =
+  c.graph.addLib(c.idgen.module, lib)
+
+func `[]`*(c: PContext, id: LibId): var TLib =
+  c.graph.getLib(id)
+
+iterator libs*(c: PContext): (LibId, var TLib)  =
+  ## Returns all ``TLib`` instances associated with `c`.
+  let pos = c.idgen.module
+  for i in 0..<c.graph.libs[pos].len:
+    yield (LibId(module: pos, index: uint32(i + 1)), c.graph.libs[pos][i])
 
 proc newTypeS*(kind: TTypeKind, c: PContext): PType =
   result = newType(kind, nextTypeId(c.idgen), getCurrOwner(c))