internal: data-oriented storage for TLib

compiler/ast/ast.nim

@@ -324,7 +324,7 @@ proc assignType*(dest, src: PType) =
   if src.sym != nil:
     if dest.sym != nil:
       dest.sym.flags.incl src.sym.flags-{sfExported}
-      if dest.sym.annex == nil: dest.sym.annex = src.sym.annex
+      if dest.sym.annex.isNil: dest.sym.annex = src.sym.annex
       dest.sym.extFlags.incl src.sym.extFlags
       if dest.sym.extname.len == 0:
         dest.sym.extname = src.sym.extname

compiler/ast/ast_types.nim

@@ -1615,6 +1615,20 @@ type
     name*: Rope
     path*: PNode              ## can be a string literal!
 
+  LibId* = object
+    ## Identifies a ``TLib`` instance. The default value means 'none'.
+    # XXX: ideally, a ``LibId`` would be a single 32-bit index into the
+    #      surrounding module, but this is not possible at the moment, because
+    #      of how aliased structural types work.
+    #
+    #        type A {.header: ... .} = int # declared in module 'A'
+    #        type B = A                    # declared in module 'B'
+    #
+    #      Here, 'B' is not a ``tyAlias`` type, but rather a ``tyInt``, with
+    #      the symbol information from 'A' (including the ``LibId``) copied
+    #      over.
+    module*: int32   ## the ID of the module the lib object is part
+    index*: uint32   ## 1-based index. Zero means 'none'
 
   CompilesId* = int ## id that is used for the caching logic within
                     ## ``system.compiles``. See the seminst module.
@@ -1633,7 +1647,6 @@ type
 
   PScope* = ref TScope
 
-  PLib* = ref TLib
   TSym* {.acyclic.} = object of TIdObj # Keep in sync with PackedSym
     ## proc and type instantiations are cached in the generic symbol
     case kind*: TSymKind
@@ -1690,7 +1703,7 @@ type
                               ## generation
     locId*: uint32            ## associates the symbol with a loc in the C code
                               ## generator. 0 means unset.
-    annex*: PLib              ## additional fields (seldom used, so we use a
+    annex*: LibId             ## additional fields (seldom used, so we use a
                               ## reference to another object to save space)
     constraint*: PNode        ## additional constraints like 'lit|result'; also
                               ## misused for the codegenDecl pragma in the hope
@@ -1892,3 +1905,5 @@ proc `comment=`*(n: PNode, a: string) =
     gconfig.comments.del(n.id)
 
 proc setUseIc*(useIc: bool) = gconfig.useIc = useIc
+
+func isNil*(id: LibId): bool = id.index == 0

compiler/backend/backends.nim

@@ -565,7 +565,8 @@ proc preprocessDynlib(graph: ModuleGraph, idgen: IdGenerator,
   #       be removed once handling of dynlib procedures and globals is fully
   #       implemented in the ``process`` iterator
   if exfDynamicLib in sym.extFlags:
-    if sym.annex.path.kind in nkStrKinds:
+    let lib = addr graph.getLib(sym.annex)
+    if lib.path.kind in nkStrKinds:
       # it's a string, no need to transform nor scan it
       discard
     else:
@@ -575,12 +576,12 @@ proc preprocessDynlib(graph: ModuleGraph, idgen: IdGenerator,
         t: MirTree
         m: SourceMap
 
-      generateCode(graph, {}, sym.annex.path, t, m)
+      generateCode(graph, {}, lib.path, t, m)
       for dep in deps(t, {}): # just ignore magics here
         if dep.kind in routineKinds + {skConst}:
           deps.add dep
 
-      sym.annex.path = generateAST(graph, idgen, sym.owner, t, m)
+      lib.path = generateAST(graph, idgen, sym.owner, t, m)
 
 proc preprocessDynlib(graph: ModuleGraph, mlist: ModuleList,
                       data: var DiscoveryData) =

compiler/backend/ccgtypes.nim

@@ -11,7 +11,7 @@
 
 # ------------------------- Name Mangling --------------------------------
 
-import compiler/sem/sighashes, compiler/modules/modulegraphs
+import compiler/sem/sighashes
 
 proc isKeyword(w: PIdent): bool =
   # Nim and C++ share some keywords

compiler/backend/cgen.nim

@@ -35,6 +35,7 @@ import
   ],
   compiler/modules/[
     magicsys,
+    modulegraphs
   ],
   compiler/front/[
     options,
@@ -145,8 +146,7 @@ proc isSimpleConst(typ: PType): bool =
 
 proc useHeader(m: BModule, sym: PSym) =
   if exfHeader in sym.extFlags:
-    assert(sym.annex != nil)
-    let str = getStr(sym.annex.path)
+    let str = getStr(m.g.graph.getLib(sym.annex).path)
     m.includeHeader(str)
 
 proc cgsym(m: BModule, name: string): Rope
@@ -628,13 +628,12 @@ include ccgexprs
 # ----------------------------- dynamic library handling -----------------
 # We don't finalize dynamic libs as the OS does this for us.
 
-proc isGetProcAddr(lib: PLib): bool =
+proc isGetProcAddr(lib: TLib): bool =
   let n = lib.path
   result = n.kind in nkCallKinds and n.typ != nil and
     n.typ.kind in {tyPointer, tyProc}
 
-proc loadDynamicLib(m: BModule, lib: PLib) =
-  assert(lib != nil)
+proc loadDynamicLib(m: BModule, lib: var TLib) =
   if not lib.generated:
     lib.generated = true
     var tmp = getTempName(m)
@@ -686,10 +685,10 @@ proc mangleDynLibProc(sym: PSym): Rope =
     result = rope(strutils.`%`("Dl_$1_", $sym.id))
 
 proc symInDynamicLib*(m: BModule, sym: PSym) =
-  var lib = sym.annex
-  let isCall = isGetProcAddr(lib)
+  var lib = addr m.g.graph.getLib(sym.annex)
+  let isCall = isGetProcAddr(lib[])
   let extname = sym.extname
-  if not isCall: loadDynamicLib(m, lib)
+  if not isCall: loadDynamicLib(m, lib[])
   let tmp = mangleDynLibProc(sym)
   # XXX: dynlib procedures should be treated as globals here (because that's
   #      what they are, really)
@@ -725,9 +724,9 @@ proc symInDynamicLib*(m: BModule, sym: PSym) =
   m.s[cfsVars].addf("$2 $1;$n", [tmp, getTypeDesc(m, sym.typ, skVar)])
 
 proc varInDynamicLib(m: BModule, sym: PSym) =
-  var lib = sym.annex
+  var lib = addr m.g.graph.getLib(sym.annex)
   let extname = sym.extname
-  loadDynamicLib(m, lib)
+  loadDynamicLib(m, lib[])
   let tmp = mangleDynLibProc(sym)
   incl(m.globals[sym].flags, lfIndirect)
   m.globals[sym].r = tmp  # from now on we only need the internal name

compiler/ic/cbackend.nim

@@ -131,6 +131,7 @@ proc generateCode*(g: ModuleGraph) =
   # XXX: these state changes were already applied during semantic analysis,
   #      but ``resetForBackend`` (unnecessarily) throws them away again
   for i in 0..high(g.packed):
+    replayLibs(g, i)
     replayBackendRoutines(g, i)
 
   var alive = computeAliveSyms(g.packed, g, g.config)

compiler/ic/ic.nim

@@ -69,6 +69,7 @@ type
     attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)]
     methodsPerType*: seq[(PackedItemId, int, PackedItemId)]
     enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
+    libs*: seq[PackedLib]
 
     emittedTypeInfo*: seq[string]
     backendFlags*: set[ModuleBackendFlag]
@@ -383,9 +384,7 @@ proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemI
     # fill the reserved slot, nothing else:
     m.types[t.uniqueId.item] = p
 
-proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib =
-  ## the plib hangs off the psym via the .annex field
-  if l.isNil: return
+proc toPackedLib(l: TLib; c: var PackedEncoder; m: var PackedModule): PackedLib =
   result.kind = l.kind
   result.generated = l.generated
   result.isOverriden = l.isOverriden
@@ -424,7 +423,8 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
     p.typ = s.typ.storeType(c, m)
     c.addMissing s.owner
     p.owner = s.owner.safeItemId(c, m)
-    p.annex = toPackedLib(s.annex, c, m)
+    if s.annex.index != 0:
+      p.annex = (toLitId(s.annex.module.FileIndex, c, m), s.annex.index)
 
     # fill the reserved slot, nothing else:
     m.syms[s.itemId.item] = p
@@ -558,6 +558,10 @@ proc storeAttachedOp*(c: var PackedEncoder; m: var PackedModule, kind: TTypeAtta
   ## Records a type-bound operator attachment action to module `m`.
   m.attachedOps.add((kind, storeTypeLater(t, c, m), storeSymLater(s, c, m)))
 
+proc storeLib*(c: var PackedEncoder, m: var PackedModule, lib: TLib) =
+  m.libs.add toPackedLib(lib, c, m)
+  flush c, m
+
 proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
   case err
   of cannotOpen:
@@ -626,6 +630,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadSeqSection attachedOpsSection, m.attachedOps
   loadSeqSection methodsPerTypeSection, m.methodsPerType
   loadSeqSection enumToStringProcsSection, m.enumToStringProcs
+  loadSeqSection libsSection, m.libs
   loadSeqSection typeInfoSection, m.emittedTypeInfo
 
   f.loadSection backendFlagsSection
@@ -692,6 +697,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection attachedOpsSection, m.attachedOps
   storeSeqSection methodsPerTypeSection, m.methodsPerType
   storeSeqSection enumToStringProcsSection, m.enumToStringProcs
+  storeSeqSection libsSection, m.libs
   storeSeqSection typeInfoSection, m.emittedTypeInfo
 
   f.storeSection backendFlagsSection
@@ -827,10 +833,15 @@ proc loadProcBody(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: in
       result = loadNodes(c, g, thisModule, tree, n0)
     inc i
 
+proc moduleIndex(c: var PackedDecoder; g: PackedModuleGraph; thisModule: int;
+                 module: LitId): int32 {.inline.} =
+  ## Returns the module ID for the stored ``FileIndex`` identified by `module`.
+  result = if module == LitId(0): thisModule.int32
+           else: toFileIndexCached(c, g, thisModule, module).int32
+
 proc moduleIndex*(c: var PackedDecoder; g: PackedModuleGraph; thisModule: int;
                   s: PackedItemId): int32 {.inline.} =
-  result = if s.module == LitId(0): thisModule.int32
-           else: toFileIndexCached(c, g, thisModule, s.module).int32
+  result = moduleIndex(c, g, thisModule, s.module)
 
 proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
                          s: PackedSym; si, item: int32): PSym =
@@ -852,14 +863,10 @@ template loadAstBodyLazy(p, field) =
     result.field = loadProcHeader(c, g, si, g[si].fromDisk.bodies, NodePos p.field)
 
 proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph;
-             si, item: int32; l: PackedLib): PLib =
-  # XXX: hack; assume a zero LitId means the PackedLib is all zero (empty)
-  if l.name.int == 0:
-    result = nil
-  else:
-    result = PLib(generated: l.generated, isOverriden: l.isOverriden,
-                  kind: l.kind, name: rope g[si].fromDisk.strings[l.name])
-    loadAstBody(l, path)
+             si: int; l: PackedLib): TLib =
+  result = TLib(generated: l.generated, isOverriden: l.isOverriden,
+                kind: l.kind, name: rope g[si].fromDisk.strings[l.name])
+  loadAstBody(l, path)
 
 proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
                        s: PackedSym; si, item: int32; result: PSym) =
@@ -869,7 +876,10 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
     loadAstBodyLazy(s, ast)
   else:
     loadAstBody(s, ast)
-  result.annex = loadLib(c, g, si, item, s.annex)
+
+  if s.annex.index != 0:
+    result.annex = LibId(module: moduleIndex(c, g, si, s.annex.module),
+                        index: s.annex.index)
 
   if s.kind in {skLet, skVar, skField, skForVar}:
     result.guard = loadSym(c, g, si, s.guard)
@@ -1094,6 +1104,12 @@ proc loadSymFromId*(config: ConfigRef, cache: IdentCache;
     setupDecoder()
     result = loadSym(decoder, g, module, id)
 
+proc loadLibs*(config: ConfigRef, cache: IdentCache,
+               g: var PackedModuleGraph, module: int): seq[TLib] =
+  setupDecoder()
+  for it in g[module].fromDisk.libs.items:
+    result.add(loadLib(decoder, g, module, it))
+
 proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; config: ConfigRef): ItemId =
   if id.module == LitId(0):
     ItemId(module: thisModule.int32, item: id.item)
@@ -1235,6 +1251,10 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
     else:
       echo "  <anon symbol?> local ID: ", i, " kind ", m.syms[i].kind
 
+  echo "all lib objects"
+  for it in m.libs.items:
+    echo "  kind: ", it.kind, " name: ", it.name, " path: ", it.path
+
   echo "symbols: ", m.syms.len, " types: ", m.types.len,
     " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len,
     " strings: ", m.strings.len, " numbers: ", m.numbers.len

compiler/ic/packed_ast.nim

@@ -61,7 +61,7 @@ type
     offset*: int
     externalName*: LitId # instead of TLoc
     extFlags*: ExternalFlags
-    annex*: PackedLib
+    annex*: tuple[module: LitId, index: uint32]
     constraint*: NodeId
 
   PackedType* = object

compiler/ic/replayer.nim

@@ -186,3 +186,11 @@ proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
   for it in mitems(g.packed[module].fromDisk.pureEnums):
     let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it))
     g.ifaces[module].pureEnums.add LazySym(id: symId, sym: nil)
+
+proc replayLibs*(g: ModuleGraph, module: int) =
+  ## Loads the packed library information for `module` into `g`.
+  # XXX: libs are not really replayed state changes...
+  if module >= g.libs.len:
+    g.libs.setLen(module + 1)
+
+  g.libs[module] = loadLibs(g.config, g.cache, g.packed, module)

compiler/ic/rodfiles.nim

@@ -93,6 +93,7 @@ type
     attachedOpsSection
     methodsPerTypeSection
     enumToStringProcsSection
+    libsSection
     typeInfoSection  # required by the backend
     backendFlagsSection
     aliveSymsSection # beware, this is stored in a `.alivesyms` file.

compiler/modules/modulegraphs.nim

@@ -29,6 +29,7 @@ import
     idents,
   ],
   compiler/utils/[
+    containers,
     pathutils,
     btrees,
     ropes,
@@ -98,6 +99,8 @@ type
     enumToStringProcs*: Table[ItemId, LazySym]
     emittedTypeInfo*: Table[string, FileIndex]
 
+    libs*: seq[seq[TLib]] ## indexed by ``LibId``
+
     startupPackedConfig*: PackedConfig
     packageSyms*: TStrTable
     deps*: IntSet # the dependency graph or potentially its transitive closure.
@@ -377,6 +380,24 @@ proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
 proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) =
   g.enumToStringProcs[t.itemId] = LazySym(sym: value)
 
+func getLib*(g: ModuleGraph, id: LibId): var TLib =
+  assert not isNil(id), "id is 'none'"
+  result = g.libs[id.module][id.index - 1]
+
+proc addLib*(g: ModuleGraph, module: int, lib: sink TLib): LibId =
+  ## Registers (adds) `lib` with the given module and returns the ID through
+  ## which the instance can be accessed from now on.
+  g.libs[module].add lib
+  # the index is 1-based, so we can directly use the length
+  result = LibId(module: module.int32, index: g.libs[module].len.uint32)
+
+proc storeLibs*(g: ModuleGraph, module: int) =
+  ## Writes the ``TLib`` instances associated with `module` to the module's
+  ## packed representation. Only relevant for IC.
+  if g.config.symbolFiles != disabledSf:
+    for lib in g.libs[module].items:
+      storeLib(g.encoders[module], g.packed[module].fromDisk, lib)
+
 iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) =
   if g.methodsPerType.contains(t.itemId):
     for it in mitems g.methodsPerType[t.itemId]:

compiler/modules/modules.nim

@@ -144,6 +144,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fr
       registerModuleById(graph, m)
       replayStateChanges(graph.packed[m.int].module, graph)
       replayGenericCacheInformation(graph, m.int)
+      replayLibs(graph, m.int)
   elif graph.isDirty(result):
     result.flags.excl sfDirty
     # reset module fields: