From 6bd130e8829d6d06f80edae114e9569532b8da7d Mon Sep 17 00:00:00 2001 From: Clonkk Date: Sun, 8 Dec 2024 17:33:14 +0100 Subject: [PATCH] Local tests --- examples/numpyarrays/results.txt | 42 +++++++++--------- scinim.nimble | 5 +++ scinim/numpyarrays.nim | 74 +++++++++++++++++++++++++++----- tests/tnumpyarrays.nim | 7 ++- 4 files changed, 92 insertions(+), 36 deletions(-) diff --git a/examples/numpyarrays/results.txt b/examples/numpyarrays/results.txt index 97f6cea..b06ec15 100644 --- a/examples/numpyarrays/results.txt +++ b/examples/numpyarrays/results.txt @@ -1,33 +1,33 @@ Python => main() CPU COUNT= 8 MAX_X= 3000 -MAX_Y= 3000 -MAX_LEN= 9000000 +MAX_Y= 4000 +MAX_LEN= 12000000 BEGIN ---------------- -[[0.26194446 0.08192133 0.55000653 ... 0.8309888 0.58720536 0.02474165] - [0.73023235 0.5942633 0.68405983 ... 0.53193958 0.06570396 0.37570736] - [0.67471879 0.30205339 0.99211628 ... 0.23007753 0.08021201 0.53655033] +[[0.70493978 0.20194409 0.36085703 ... 0.00690316 0.193097 0.23888245] + [0.64239022 0.85585042 0.3957121 ... 0.37209593 0.46811377 0.6701858 ] + [0.75608895 0.131218 0.42835299 ... 0.41229778 0.80615316 0.39378482] ... - [0.53500795 0.67878598 0.82899082 ... 0.6583351 0.82552753 0.13155744] - [0.5549234 0.55656235 0.86248167 ... 0.72473359 0.18438149 0.60065313] - [0.86027365 0.24390531 0.57120545 ... 0.69851604 0.81229157 0.07272132]] + [0.48456984 0.58292855 0.40547578 ... 0.63544508 0.45781191 0.71426714] + [0.79431227 0.03892199 0.42968725 ... 0.83385262 0.38061308 0.70092219] + [0.51775866 0.20859028 0.56165316 ... 0.03074889 0.96434193 0.41807707]] 1) -res= 3477241.833174428 -Python loop took : 3.6015096540213563 seconds -pyres= 3477241.833174428 -Python measured native loop took : 0.018548287014709786 seconds -Nim measured native loop took : 18.52 ms -res= 3477241.833174428 +res= 4635196.899425893 +Python loop took : 5.607424815010745 seconds +pyres= 4635196.899425893 +Python measured native loop took : 0.02337951000663452 seconds +Nim measured native loop took : 23.357 ms +res= 4635196.899425893 2) Showing in-place mod -[0.26194446 0.08192133 0.55000653] -[123. -5. 0.55000653] +[0.70493978 0.20194409 0.36085703] +[123. -5. 0.36085703] 3) Comparing for loops -normalForOp: 0.03445864300010726 seconds -indexedOp: 0.04115106599056162 seconds -parallelForOp: 0.00984231400070712 seconds -parallelIndexedForOp: 0.024510202987585217 seconds -Native python for: 5.1321438050072175 seconds +normalForOp: 0.04339129300205968 seconds +indexedOp: 0.051275824022013694 seconds +parallelForOp: 0.012469089997466654 seconds +parallelIndexedForOp: 0.03665532698505558 seconds +Native python for: 7.6476721960061695 seconds False False True diff --git a/scinim.nimble b/scinim.nimble index 8777b9c..3f26384 100644 --- a/scinim.nimble +++ b/scinim.nimble @@ -13,8 +13,13 @@ requires "threading" requires "arraymancer >= 0.7.32" requires "polynumeric >= 0.2.0" requires "nimpy >= 0.2.0" +requires "print" task test, "Run all tests": echo "Running tests command" exec "nim cpp -r --mm:arc tests/tnumpyarrays.nim" exec "nim cpp -r --mm:orc tests/tnumpyarrays.nim" + exec "nim c -r tests/tnumpyarrays.nim" + exec "nim c -r --gc:orc tests/tnumpyarrays.nim" + exec "nim cpp -r tests/tnumpyarrays.nim" + exec "nim cpp -r --gc:orc tests/tnumpyarrays.nim" diff --git a/scinim/numpyarrays.nim b/scinim/numpyarrays.nim index cb43df3..af1ab9b 100644 --- a/scinim/numpyarrays.nim +++ b/scinim/numpyarrays.nim @@ -4,8 +4,10 @@ import threading/smartptrs import arraymancer -import nimpy +import nimpy {.all.} import nimpy/[raw_buffers, py_types, py_utils] +import nimpy/py_lib as lib +import ./bindings {.push gcsafe.} @@ -141,12 +143,12 @@ proc initNumpyArray*[T](ar: sink PyObject): NumpyArray[T] = for i in 0 ..< result.pyBuf.raw.ndim: let dimsize = shapear[i].int # py_ssize_t is csize result.shape.add dimsize + result.len = result.shape.foldl(a * b, 1) result.data = cast[ptr UncheckedArray[T]](result.pyBuf.raw.buf) proc pyValueToNim*[T: SomeNumber](v: PPyObject, o: var NumpyArray[T]) {.inline.} = - incRef(v) - var vv = PyObject(rawPyObj: v) + var vv = newPyObject(v) o = initNumpyArray[T](vv) proc isContiguous*[T](ar: NumpyArray[T]) : bool = @@ -173,14 +175,64 @@ proc asNumpyArray*[T](ar: sink PyObject): NumpyArray[T] = return initNumpyArray[T](ar) proc ndArrayFromPtr*[T](t: ptr T, shape: seq[int]): NumpyArray[T] = - let np = pyImport("numpy") - let py_array_type = dtype(T) - # Just a trick to force an initialization of a Numpy Array of the correct size - result = asNumpyArray[T]( - nimpy.callMethod(np, "zeros", shape, py_array_type) - ) - # copyMem the data - var bsizes = result.len*(sizeof(T) div sizeof(uint8)) + # let np = pyImport("numpy") + # let py_array_type = dtype(T) + # # Just a trick to force an initialization of a Numpy Array of the correct size + # result = asNumpyArray[T]( + # nimpy.callMethod(np, "zeros", shape, py_array_type) + # ) + # var bsizes = result.len*(sizeof(T) div sizeof(uint8)) + # copyMem(addr(result.data[0]), t, bsizes) + + # let + # nd = len(shape) + # dims = shape.map(x => csize_t(x)) + # pdims = addr dims[0] + # var tmp = NumpyArrayZeros[T](nd, pdims) + # result = asNumpyArray[T](tmp[]) + + let + nelem = shape.foldl(a * b, 1) + bsizes = nelem*(sizeof(T) div sizeof(uint8)) + + # # copyMem the data + # # The memory used by a numpy is a sys.getsizeof(array) - array.nbytes. + # # Assuming a first order solution : C + ndim*x. + # # Testing for different ndim with identical array size : {4: 160, 3: 144, 5: 176, 2: 128, 1: 112} + # # This gives => x=16 and C=96 + # let total_size = bsizes + len(shape)*16 + 96 + # var + # v = pyAlloc(total_size) + # nlen = shape.foldl(a * b, 1) + # strides = newSeq[int](len(shape)) + # size = 1 + + # for i in countdown(shape.len() - 1, 0): + # strides[i] = size + # size *= shape[i] + + # echo "nlen=", nlen + # echo "shape=", shape + # echo "strides=", strides + # echo pyLib.isNil() + + # let + # ppylen = nimValueToPy(nlen) + # ppyshape = nimValueToPy(shape) + # ppystrides = nimValueToPy(strides) + + # # var + # # r = pyLib.PyObject_SetAttrString(v, "len", pylen) + # # r = r + pyLib.PyObject_SetAttrString(v, "shape", pyshape) + # # r = r + pyLib.PyObject_SetAttrString(v, "strides", pystrides) + # # if unlikely r != 0: raisePythonError() + + # var vv = newPyObject(v) + # setAttrAux(vv, "len", ppylen) + # setAttrAux(vv, "shape", ppyshape) + # setAttrAux(vv, "strides", ppystrides) + + # result = asNumpyArray[T](vv) copyMem(addr(result.data[0]), t, bsizes) proc ndArrayFromPtr*[T](t: ptr UncheckedArray[T], shape: seq[int]): NumpyArray[T] = diff --git a/tests/tnumpyarrays.nim b/tests/tnumpyarrays.nim index 9ac45a5..3644506 100644 --- a/tests/tnumpyarrays.nim +++ b/tests/tnumpyarrays.nim @@ -76,7 +76,6 @@ proc test(arg: tuple[s: string]) = when isMainModule: test((s: "toTensor, toNdArray in main thread")) # Disable for now - var thr: Thread[tuple[s: string]] - createThread(thr, test, (s: "toTensor, toNdArray in external thread")) - joinThread(thr) - + # var thr: Thread[tuple[s: string]] + # createThread(thr, test, (s: "toTensor, toNdArray in external thread")) + # joinThread(thr)