[Nim] Bfbs Nim Generator (google#7534)

* Bfbs Nim Generator

* Remove commented out tests

* add missing line to idl.h

* Commit python reflection changes

* Commit python reflection changes and move tests

* Remove default string addition

* Move tests to python file

* Fix element size check when element is table

* remove whitespace changes

* add element_type docs and commit further to namer and remove kkeep

* Bfbs Nim Generator

* Remove commented out tests

* add missing line to idl.h

* Commit python reflection changes

* Commit python reflection changes and move tests

* Remove default string addition

* Move tests to python file

* Fix element size check when element is table

* remove whitespace changes

* add element_type docs and commit further to namer and remove kkeep

* remove unused variables

* added tests to ci

* added tests to ci

* fixes

* Added reflection type Field, Variable to namer

* Moved reflection namer impl to bfbsnamer

* Remove whitespace at end of line

* Added nim to generated code

* Revert whitespace removal

Co-authored-by: Derek Bailey <derekbailey@google.com>

.github/labeler.yml

@@ -18,6 +18,12 @@ swift:
   - tests/swift/**
   - src/idl_gen_swift.cpp
 
+nim:
+  - '**/*.nim'
+  - nim/**/*
+  - src/idl_gen_nim.cpp
+  - src/bfbs_gen_nim.cpp
+
 javascript:
   - '**/*.js'
   - src/idl_gen_ts.cpp
@@ -61,7 +67,7 @@ rust:
   - '**/*.rs'
   - rust/**/*
   - src/idl_gen_rust.cpp
-  
+
 dart:
   - '**/*.dart'
   - src/idl_gen_dart.cpp
@@ -88,4 +94,4 @@ CI:
 
 grpc:
   - grpc/**/*
-  - src/idl_gen_grpc.cpp
+  - src/idl_gen_grpc.cpp

.github/workflows/build.yml

@@ -477,6 +477,22 @@ jobs:
         working-directory: tests
         run: bash DartTest.sh
 
+  build-nim:
+    name: Build Nim
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: flatc
+      # FIXME: make test script not rely on flatc
+      run: cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DFLATBUFFERS_BUILD_TESTS=OFF -DFLATBUFFERS_INSTALL=OFF -DFLATBUFFERS_BUILD_FLATLIB=OFF -DFLATBUFFERS_BUILD_FLATHASH=OFF . && make -j
+    - uses: jiro4989/setup-nim-action@v1
+    - name: install library
+      working-directory: nim
+      run: nimble -y develop
+    - name: test
+      working-directory: tests/nim
+      run: python3 testnim.py
+
   release-digests:
     if: startsWith(github.ref, 'refs/tags/')
     needs: [build-linux, build-windows, build-mac-intel, build-mac-universal]

CMakeLists.txt

@@ -182,13 +182,15 @@ set(FlatBuffers_Compiler_SRCS
   src/flatc_main.cpp
   src/bfbs_gen.h
   src/bfbs_gen_lua.h
+  src/bfbs_gen_nim.h
   src/bfbs_namer.h
   include/flatbuffers/code_generators.h
   src/binary_annotator.h
   src/binary_annotator.cpp
   src/annotated_binary_text_gen.h
   src/annotated_binary_text_gen.cpp
   src/bfbs_gen_lua.cpp
+  src/bfbs_gen_nim.cpp
   src/code_generators.cpp
   grpc/src/compiler/schema_interface.h
   grpc/src/compiler/cpp_generator.h

include/flatbuffers/idl.h

@@ -661,6 +661,7 @@ struct IDLOptions {
     kRust = 1 << 14,
     kKotlin = 1 << 15,
     kSwift = 1 << 16,
+    kNim = 1 << 17,
     kMAX
   };
 

nim/flatbuffers.nimble

@@ -0,0 +1,7 @@
+version     = "2.0.8"
+author      = "flatbuffers"
+description = "Flatbuffers"
+license     = "Apache 2.0"
+srcDir      = "flatbuffers"
+
+requires "nim >= 1.4.0"

nim/flatbuffers/flatbuffers.nim

@@ -0,0 +1,7 @@
+import
+   src/[
+      builder,
+      struct,
+      table
+   ]
+export flatbuffers.builder, flatbuffers.table, flatbuffers.struct

nim/flatbuffers/src/builder.nim

@@ -0,0 +1,262 @@
+import math
+import table
+
+
+const MAX_BUFFER_SIZE* = 2^31
+
+
+type Builder* = ref object of RootObj
+   bytes*: seq[byte]
+   minalign*: int
+   current_vtable*: seq[uoffset]
+   objectEnd*: uoffset
+   vtables*: seq[uoffset] #?
+   head*: uoffset
+   nested*: bool
+   finished*: bool
+   vectorNumElems*: uoffset
+
+using this: var Builder
+
+func newBuilder*(size: int): Builder =
+   result = new Builder
+   result.bytes.setLen(size)
+   result.minalign = 1
+   result.head = size.uoffset
+   result.nested = false
+   result.finished = false
+   result.vectorNumElems = 0
+
+proc FinishedBytes*(this): seq[byte] =
+   if not this.finished:
+      quit("Builder not finished, Incorrect use of FinishedBytes(): must call 'Finish' first.")
+   result = this.bytes[this.head..^1]
+
+proc Output*(this): seq[byte] =
+   if not this.finished:
+      quit("Builder not finished, Incorrect use of Output(): must call 'Finish' first.")
+
+   result = this.bytes[this.head..^1]
+
+func Offset*(this): uoffset =
+   result = this.bytes.len.uoffset - this.head
+
+proc StartObject*(this; numfields: int) =
+   if this.nested:
+      quit("builder is nested")
+
+   this.current_vtable.setLen(numfields)
+   for i in this.current_vtable.mitems():
+      i = 0
+   this.objectEnd = this.Offset()
+   this.nested = true
+
+proc GrowByteBuffer*(this) =
+   if this.bytes.len == MAX_BUFFER_SIZE:
+      quit("flatbuffers: cannot grow buffer beyond 2 gigabytes")
+   let oldLen = this.bytes.len
+   var newLen = min(this.bytes.len * 2, MAX_BUFFER_SIZE)
+   if newLen == 0:
+      newLen = 1
+   this.bytes.setLen(newLen)
+   var j = this.bytes.len - 1
+   while j >= 0:
+      if j >= newLen - oldLen:
+         this.bytes[j] = this.bytes[j - (newLen - oldLen)]
+      else:
+         this.bytes[j] = 0
+      dec(j)
+
+proc Place*[T](this; x: T) =
+   this.head -= uoffset x.sizeof
+   WriteVal(this.bytes, this.head, x)
+
+func Pad*(this; n: int) =
+   for i in 0..<n:
+      this.Place(0.byte)
+
+proc Prep*(this; size: int; additionalBytes: int) =
+   if size > this.minalign:
+      this.minalign = size
+   var alignsize = (not (this.bytes.len - this.head.int + additionalBytes)) + 1
+   alignsize = alignsize and (size - 1)
+
+   while this.head.int < alignsize + size + additionalBytes:
+      let oldbufSize = this.bytes.len
+      this.GrowByteBuffer()
+      this.head = (this.head.int + this.bytes.len - oldbufSize).uoffset
+   this.Pad(alignsize)
+
+proc PrependOffsetRelative*[T: Offsets](this; off: T) =
+   when T is voffset:
+      this.Prep(T.sizeof, 0)
+      if not off.uoffset <= this.Offset:
+         quit("flatbuffers: Offset arithmetic error.")
+      this.Place(off)
+   else:
+      this.Prep(T.sizeof, 0)
+      if not off.uoffset <= this.Offset:
+         quit("flatbuffers: Offset arithmetic error.")
+      let off2: T = this.Offset.T - off + sizeof(T).T
+      this.Place(off2)
+
+
+proc Prepend*[T](this; x: T) =
+   this.Prep(x.sizeof, 0)
+   this.Place(x)
+
+proc Slot*(this; slotnum: int) =
+   this.current_vtable[slotnum] = this.Offset
+
+proc PrependSlot*[T](this; o: int; x, d: T) =
+   if x != d:
+      when T is uoffset or T is soffset or T is voffset:
+         this.PrependOffsetRelative(x)
+      else:
+         this.Prepend(x)
+      this.Slot(o)
+
+proc AssertStuctInline(this; obj: uoffset) =
+   if obj != this.Offset:
+      quit("flatbuffers: Tried to write a Struct at an Offset that is different from the current Offset of the Builder.")
+
+proc PrependStructSlot*(this; o: int; x: uoffset; d: uoffset) =
+   if x != d:
+      this.AssertStuctInline(x)
+      this.Slot(o)
+
+proc Add*[T](this; n: T) =
+   this.Prep(T.sizeof, 0)
+   WriteVal(this.bytes, this.head, n)
+
+proc VtableEqual*(a: seq[uoffset]; objectStart: uoffset; b: seq[byte]): bool =
+   if a.len * voffset.sizeof != b.len:
+      return false
+
+   var i = 0
+   while i < a.len:
+      var seq = b[i * voffset.sizeof..<(i + 1) * voffset.sizeof]
+      let x = GetVal[voffset](addr seq)
+
+      if x == 0 and a[i] == 0:
+         inc i
+         continue
+
+      let y = objectStart.soffset - a[i].soffset
+      if x.soffset != y:
+         return false
+      inc i
+   return true
+
+proc WriteVtable*(this): uoffset =
+   this.PrependOffsetRelative(0.soffset)
+
+   let objectOffset = this.Offset
+   var existingVtable = uoffset 0
+
+   var i = this.current_vtable.len - 1
+   while i >= 0 and this.current_vtable[i] == 0: dec i
+
+   this.current_vtable = this.current_vtable[0..i]
+   for i in countdown(this.vtables.len - 1, 0):
+      let
+         vt2Offset: uoffset = this.vtables[i]
+         vt2Start: int = this.bytes.len - int vt2Offset
+
+      var seq = this.bytes[vt2Start..<this.bytes.len]
+      let
+         vt2Len = GetVal[voffset](addr seq)
+         metadata = 2 * voffset.sizeof # VtableMetadataFields * SizeVOffsetT
+         vt2End = vt2Start + vt2Len.int
+         vt2 = this.bytes[this.bytes.len - vt2Offset.int + metadata..<vt2End]
+
+      if VtableEqual(this.current_vtable, objectOffset, vt2):
+         existingVtable = vt2Offset
+         break
+
+   if existingVtable == 0:
+      for i in countdown(this.current_vtable.len - 1, 0):
+         var off: uoffset
+         if this.current_vtable[i] != 0:
+            off = objectOffset - this.current_vtable[i]
+
+         this.PrependOffsetRelative(off.voffset)
+
+      let objectSize = objectOffset - this.objectEnd
+      this.PrependOffsetRelative(objectSize.voffset)
+
+      let vBytes = (this.current_vtable.len + 2) * voffset.sizeof
+      this.PrependOffsetRelative(vBytes.voffset)
+
+      let objectStart: uoffset = (this.bytes.len.uoffset - objectOffset)
+      WriteVal(this.bytes, objectStart, (this.Offset - objectOffset).soffset)
+      this.vtables.add this.Offset
+   else:
+      let objectStart: uoffset = this.bytes.len.uoffset - objectOffset
+      this.head = objectStart
+      WriteVal(this.bytes, this.head,
+         (existingVtable.soffset - objectOffset.soffset))
+
+      this.current_vtable = @[]
+   result = objectOffset
+
+proc EndObject*(this): uoffset =
+   if not this.nested:
+      quit("builder is not nested")
+   result = this.WriteVtable()
+   this.nested = false
+
+proc End*(this: var Builder): uoffset =
+   result = this.EndObject()
+
+proc StartVector*(this; elemSize: int; numElems: uoffset;
+      alignment: int) =
+   if this.nested:
+      quit("builder is nested")
+   this.nested = true
+   this.vectorNumElems = numElems
+   this.Prep(sizeof(uint32), elemSize * numElems.int)
+   this.Prep(alignment, elemSize * numElems.int)
+
+proc EndVector*(this): uoffset =
+   if not this.nested:
+      quit("builder is not nested")
+   this.nested = false
+   this.Place(this.vectorNumElems)
+   this.vectorNumElems = 0
+   result = this.Offset
+
+proc getChars*(str: seq[byte]): string =
+   var bytes = str
+   result = GetVal[string](addr bytes)
+
+proc getBytes*(str: string | cstring): seq[byte] =
+   for chr in str:
+      result.add byte chr
+   result.add byte 0
+
+proc Create*[T](this; s: T): uoffset = # Both CreateString and CreateByteVector functionality
+   if this.nested:
+      quit("builder is nested")
+   this.nested = true
+   when T is cstring or T is string:
+      let x = s.getBytes()
+      let l = x.len.uoffset
+      this.vectorNumElems = l-1
+   else:
+      let x = s
+      let l = x.len.uoffset
+      this.vectorNumElems = l
+   this.Prep(uoffset.sizeof, l.int * byte.sizeof)
+   this.head -= l
+   this.bytes[this.head..<this.head+l] = x
+   result = this.EndVector()
+
+proc Finish*(this; rootTable: uoffset) =
+   if this.nested:
+      quit("builder is nested")
+   this.nested = true
+
+   this.Prep(this.minalign, uoffset.sizeof)
+   this.PrependOffsetRelative(rootTable)
+   this.finished = true

nim/flatbuffers/src/endian.nim

@@ -0,0 +1,12 @@
+template swapEndian*(outp, inp: pointer, size: int) =
+   var i = cast[cstring](inp)
+   var o = cast[cstring](outp)
+   for x in 0..<size:
+      o[x] = i[(0..<size).len - x - 1]
+
+when system.cpuEndian == bigEndian:
+   func littleEndianX*(outp, inp: pointer, size: int) {.inline.} = swapEndian(outp, inp, size)
+   func bigEndianX*(outp, inp: pointer, size: int) {.inline.} = copyMem(outp, inp, size)
+else:
+   func littleEndianX*(outp, inp: pointer, size: int) {.inline.} = copyMem(outp, inp, size)
+   func bigEndianX*(outp, inp: pointer, size: int) {.inline.} = swapEndian(outp, inp, size)

nim/flatbuffers/src/struct.nim

@@ -0,0 +1,24 @@
+import table
+
+
+type FlatObj* {.inheritable.} = object
+   tab*: Vtable
+
+func Table*(this: var FlatObj): Vtable = this.tab
+
+func Init*(this: var FlatObj; buf: seq[byte]; i: uoffset) =
+   this.tab.Bytes = buf
+   this.tab.Pos = i
+
+# Cant define it in table.nim since it needs FlatObj and Init
+func GetUnion*[T: FlatObj](this: var Vtable; off: uoffset): T =
+   result.Init(this.Bytes, this.Indirect(off))
+
+func GetRootAs*(result: var FlatObj; buf: seq[byte]; offset: uoffset) =
+   var
+      vtable = Vtable(Bytes: buf[offset..^1], Pos: offset)
+      n = Get[uoffset](vtable, offset)
+   result.Init(buf, n+offset)
+
+func GetRootAs*(result: var FlatObj; buf: string; offset: uoffset) =
+   result.GetRootAs(cast[seq[byte]](buf), offset)