Local tests

examples/numpyarrays/results.txt

@@ -1,33 +1,33 @@
 Python => main()
 CPU COUNT= 8
 MAX_X= 3000
-MAX_Y= 3000
-MAX_LEN= 9000000
+MAX_Y= 4000
+MAX_LEN= 12000000
 BEGIN
 ----------------
-[[0.26194446 0.08192133 0.55000653 ... 0.8309888  0.58720536 0.02474165]
- [0.73023235 0.5942633  0.68405983 ... 0.53193958 0.06570396 0.37570736]
- [0.67471879 0.30205339 0.99211628 ... 0.23007753 0.08021201 0.53655033]
+[[0.70493978 0.20194409 0.36085703 ... 0.00690316 0.193097   0.23888245]
+ [0.64239022 0.85585042 0.3957121  ... 0.37209593 0.46811377 0.6701858 ]
+ [0.75608895 0.131218   0.42835299 ... 0.41229778 0.80615316 0.39378482]
  ...
- [0.53500795 0.67878598 0.82899082 ... 0.6583351  0.82552753 0.13155744]
- [0.5549234  0.55656235 0.86248167 ... 0.72473359 0.18438149 0.60065313]
- [0.86027365 0.24390531 0.57120545 ... 0.69851604 0.81229157 0.07272132]]
+ [0.48456984 0.58292855 0.40547578 ... 0.63544508 0.45781191 0.71426714]
+ [0.79431227 0.03892199 0.42968725 ... 0.83385262 0.38061308 0.70092219]
+ [0.51775866 0.20859028 0.56165316 ... 0.03074889 0.96434193 0.41807707]]
 1)
-res= 3477241.833174428
-Python loop took :  3.6015096540213563  seconds
-pyres= 3477241.833174428
-Python measured native loop took :  0.018548287014709786  seconds
-Nim measured native loop took :  18.52  ms
-res= 3477241.833174428
+res= 4635196.899425893
+Python loop took :  5.607424815010745  seconds
+pyres= 4635196.899425893
+Python measured native loop took :  0.02337951000663452  seconds
+Nim measured native loop took :  23.357  ms
+res= 4635196.899425893
 2) Showing in-place mod
-[0.26194446 0.08192133 0.55000653]
-[123.          -5.           0.55000653]
+[0.70493978 0.20194409 0.36085703]
+[123.          -5.           0.36085703]
 3) Comparing for loops
-normalForOp:  0.03445864300010726  seconds
-indexedOp:  0.04115106599056162  seconds
-parallelForOp:  0.00984231400070712  seconds
-parallelIndexedForOp:  0.024510202987585217  seconds
-Native python for:  5.1321438050072175  seconds
+normalForOp:  0.04339129300205968  seconds
+indexedOp:  0.051275824022013694  seconds
+parallelForOp:  0.012469089997466654  seconds
+parallelIndexedForOp:  0.03665532698505558  seconds
+Native python for:  7.6476721960061695  seconds
 False
 False
 True

scinim.nimble

@@ -13,8 +13,13 @@ requires "threading"
 requires "arraymancer >= 0.7.32"
 requires "polynumeric >= 0.2.0"
 requires "nimpy >= 0.2.0"
+requires "print"
 
 task test, "Run all tests":
   echo "Running tests command"
   exec "nim cpp -r --mm:arc tests/tnumpyarrays.nim"
   exec "nim cpp -r --mm:orc tests/tnumpyarrays.nim"
+  exec "nim c -r tests/tnumpyarrays.nim"
+  exec "nim c -r --gc:orc tests/tnumpyarrays.nim"
+  exec "nim cpp -r tests/tnumpyarrays.nim"
+  exec "nim cpp -r --gc:orc tests/tnumpyarrays.nim"

scinim/numpyarrays.nim

@@ -4,8 +4,10 @@ import threading/smartptrs
 
 import arraymancer
 
-import nimpy
+import nimpy {.all.}
 import nimpy/[raw_buffers, py_types, py_utils]
+import nimpy/py_lib as lib
+import ./bindings
 
 {.push gcsafe.}
 
@@ -141,12 +143,12 @@ proc initNumpyArray*[T](ar: sink PyObject): NumpyArray[T] =
   for i in 0 ..< result.pyBuf.raw.ndim:
     let dimsize = shapear[i].int # py_ssize_t is csize
     result.shape.add dimsize
+
   result.len = result.shape.foldl(a * b, 1)
   result.data = cast[ptr UncheckedArray[T]](result.pyBuf.raw.buf)
 
 proc pyValueToNim*[T: SomeNumber](v: PPyObject, o: var NumpyArray[T]) {.inline.} =
-  incRef(v)
-  var vv = PyObject(rawPyObj: v)
+  var vv = newPyObject(v)
   o = initNumpyArray[T](vv)
 
 proc isContiguous*[T](ar: NumpyArray[T]) : bool =
@@ -173,14 +175,64 @@ proc asNumpyArray*[T](ar: sink PyObject): NumpyArray[T] =
     return initNumpyArray[T](ar)
 
 proc ndArrayFromPtr*[T](t: ptr T, shape: seq[int]): NumpyArray[T] =
-  let np = pyImport("numpy")
-  let py_array_type = dtype(T)
-  # Just a trick to force an initialization of a Numpy Array of the correct size
-  result = asNumpyArray[T](
-    nimpy.callMethod(np, "zeros", shape, py_array_type)
-  )
-  # copyMem the data
-  var bsizes = result.len*(sizeof(T) div sizeof(uint8))
+  # let np = pyImport("numpy")
+  # let py_array_type = dtype(T)
+  # # Just a trick to force an initialization of a Numpy Array of the correct size
+  # result = asNumpyArray[T](
+  #   nimpy.callMethod(np, "zeros", shape, py_array_type)
+  # )
+  # var bsizes = result.len*(sizeof(T) div sizeof(uint8))
+  # copyMem(addr(result.data[0]), t, bsizes)
+
+  # let
+  #   nd = len(shape)
+  #   dims = shape.map(x => csize_t(x))
+  #   pdims = addr dims[0]
+  # var tmp = NumpyArrayZeros[T](nd, pdims)
+  # result = asNumpyArray[T](tmp[])
+
+  let
+    nelem = shape.foldl(a * b, 1)
+    bsizes = nelem*(sizeof(T) div sizeof(uint8))
+
+  # # copyMem the data
+  # # The memory used by a numpy is a sys.getsizeof(array) - array.nbytes.
+  # # Assuming a first order solution : C + ndim*x.
+  # # Testing for different ndim with identical array size : {4: 160, 3: 144, 5: 176, 2: 128, 1: 112}
+  # # This gives => x=16 and C=96
+  # let total_size = bsizes + len(shape)*16 + 96
+  # var
+  #   v = pyAlloc(total_size)
+  #   nlen = shape.foldl(a * b, 1)
+  #   strides = newSeq[int](len(shape))
+  #   size = 1
+
+  # for i in countdown(shape.len() - 1, 0):
+  #   strides[i] = size
+  #   size *= shape[i]
+
+  # echo "nlen=", nlen
+  # echo "shape=", shape
+  # echo "strides=", strides
+  # echo pyLib.isNil()
+
+  # let
+  #   ppylen     = nimValueToPy(nlen)
+  #   ppyshape   = nimValueToPy(shape)
+  #   ppystrides = nimValueToPy(strides)
+
+  # # var
+  # #   r = pyLib.PyObject_SetAttrString(v, "len", pylen)
+  # # r = r + pyLib.PyObject_SetAttrString(v, "shape", pyshape)
+  # # r = r + pyLib.PyObject_SetAttrString(v, "strides", pystrides)
+  # # if unlikely r != 0: raisePythonError()
+
+  # var vv = newPyObject(v)
+  # setAttrAux(vv, "len", ppylen)
+  # setAttrAux(vv, "shape", ppyshape)
+  # setAttrAux(vv, "strides", ppystrides)
+
+  # result = asNumpyArray[T](vv)
   copyMem(addr(result.data[0]), t, bsizes)
 
 proc ndArrayFromPtr*[T](t: ptr UncheckedArray[T], shape: seq[int]): NumpyArray[T] =

tests/tnumpyarrays.nim

@@ -76,7 +76,6 @@ proc test(arg: tuple[s: string]) =
 when isMainModule:
   test((s: "toTensor, toNdArray in main thread"))
   # Disable for now
-  var thr: Thread[tuple[s: string]]
-  createThread(thr, test, (s: "toTensor, toNdArray in external thread"))
-  joinThread(thr)
-
+  # var thr: Thread[tuple[s: string]]
+  # createThread(thr, test, (s: "toTensor, toNdArray in external thread"))
+  # joinThread(thr)