From 7f1d261da4e7f98769079c78548d05eccebbb56f Mon Sep 17 00:00:00 2001 From: Ianna Osborne Date: Thu, 2 May 2024 19:07:46 +0200 Subject: [PATCH] fix: intermittent segfault in PyPy (#3089) * debug: add debugging flag * test: debug * fix: allocate memory for 2d arrays in tests * fix: remove legacy parameter from api * debug: exclude some tests that rely on 2d array parameters in cpu kernels * debug: exclude cpu kernel unt tests with 2d arrays as parameters * fix: typo * test: remove coverage for pypy * fix: restore tests * fix: restore cpu kernel unit tests * fix: remove debugger * fix: restore tests * fix: add more tests --------- Co-authored-by: Jim Pivarski --- .DS_Store | Bin 8196 -> 8196 bytes .github/workflows/test.yml | 9 +- .../awkward_UnionArray_flatten_length.cpp | 6 +- .../src/cpu-kernels/awkward_unique_ranges.cpp | 21 ----- .../awkward_unique_ranges_bool.cpp | 1 - awkward-cpp/src/cpu-kernels/kernel-utils.cpp | 2 +- dev/generate-tests.py | 80 +++++++++++++++--- kernel-specification.yml | 15 +--- kernel-test-data.json | 24 ++---- src/awkward/contents/numpyarray.py | 2 - 10 files changed, 87 insertions(+), 73 deletions(-) diff --git a/.DS_Store b/.DS_Store index 40072e236494d6318ae89fda2260a2129e9dd017..f0c6683eef7b709a38f347afd2bb5a398c6cbc52 100644 GIT binary patch delta 97 zcmZp1XmOa}&nUMsU^hRb++-es{K>8Y;v8uy#mPze`8k`*1tb{xnHf?TQW?tNg583X dSU6!k#?AaM<873^c#oFz7wabkn-W_F2hESrr*6&Rr+ FUI6wT5?ufQ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 47a025756d..a86ff5940a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -139,16 +139,21 @@ jobs: - name: Test CPU kernels with explicitly defined values run: python -m pytest -vv -rs awkward-cpp/tests-cpu-kernels-explicit - - name: Test non-kernels + - name: Test non-kernels (Python) run: >- python -m pytest -vv -rs tests --cov=awkward --cov-report=term --cov-report=xml + if: startsWith(matrix.python-version, '3.') + + - name: Test non-kernels (PyPy) + run: >- + python -m pytest -vv -rs tests + if: startsWith(matrix.python-version, 'pypy') - name: Upload Codecov results if: (matrix.python-version == '3.9') && (matrix.runs-on == 'ubuntu-latest') uses: codecov/codecov-action@v4 - Linux-ROOT: runs-on: ubuntu-20.04 diff --git a/awkward-cpp/src/cpu-kernels/awkward_UnionArray_flatten_length.cpp b/awkward-cpp/src/cpu-kernels/awkward_UnionArray_flatten_length.cpp index 1afe377de1..ae49efde44 100644 --- a/awkward-cpp/src/cpu-kernels/awkward_UnionArray_flatten_length.cpp +++ b/awkward-cpp/src/cpu-kernels/awkward_UnionArray_flatten_length.cpp @@ -13,11 +13,11 @@ ERROR awkward_UnionArray_flatten_length( T** offsetsraws) { *total_length = 0; for (int64_t i = 0; i < length; i++) { - FROMTAGS tag = fromtags[i]; - FROMINDEX idx = fromindex[i]; + int64_t tag = (int64_t)fromtags[i]; + int64_t idx = (int64_t)fromindex[i]; T start = offsetsraws[tag][idx]; T stop = offsetsraws[tag][idx + 1]; - *total_length = *total_length + (stop - start); + *total_length = *total_length + (int64_t)(stop - start); } return success(); } diff --git a/awkward-cpp/src/cpu-kernels/awkward_unique_ranges.cpp b/awkward-cpp/src/cpu-kernels/awkward_unique_ranges.cpp index 6b8fa08024..a9edf83553 100644 --- a/awkward-cpp/src/cpu-kernels/awkward_unique_ranges.cpp +++ b/awkward-cpp/src/cpu-kernels/awkward_unique_ranges.cpp @@ -7,7 +7,6 @@ template ERROR awkward_unique_ranges( T* toptr, - int64_t /* length */, // FIXME: this argument is not needed const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { @@ -28,13 +27,11 @@ ERROR awkward_unique_ranges( ERROR awkward_unique_ranges_int8( int8_t* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); @@ -42,13 +39,11 @@ ERROR awkward_unique_ranges_int8( ERROR awkward_unique_ranges_uint8( uint8_t* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); @@ -56,13 +51,11 @@ ERROR awkward_unique_ranges_uint8( ERROR awkward_unique_ranges_int16( int16_t* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); @@ -70,13 +63,11 @@ ERROR awkward_unique_ranges_int16( ERROR awkward_unique_ranges_uint16( uint16_t* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); @@ -84,13 +75,11 @@ ERROR awkward_unique_ranges_uint16( ERROR awkward_unique_ranges_int32( int32_t* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); @@ -98,13 +87,11 @@ ERROR awkward_unique_ranges_int32( ERROR awkward_unique_ranges_uint32( uint32_t* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); @@ -112,13 +99,11 @@ ERROR awkward_unique_ranges_uint32( ERROR awkward_unique_ranges_int64( int64_t* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); @@ -126,13 +111,11 @@ ERROR awkward_unique_ranges_int64( ERROR awkward_unique_ranges_uint64( uint64_t* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); @@ -140,13 +123,11 @@ ERROR awkward_unique_ranges_uint64( ERROR awkward_unique_ranges_float32( float* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); @@ -154,13 +135,11 @@ ERROR awkward_unique_ranges_float32( ERROR awkward_unique_ranges_float64( double* toptr, - int64_t length, const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { return awkward_unique_ranges( toptr, - length, fromoffsets, offsetslength, tooffsets); diff --git a/awkward-cpp/src/cpu-kernels/awkward_unique_ranges_bool.cpp b/awkward-cpp/src/cpu-kernels/awkward_unique_ranges_bool.cpp index 4636338041..6ddc9d8268 100644 --- a/awkward-cpp/src/cpu-kernels/awkward_unique_ranges_bool.cpp +++ b/awkward-cpp/src/cpu-kernels/awkward_unique_ranges_bool.cpp @@ -6,7 +6,6 @@ ERROR awkward_unique_ranges_bool( bool* toptr, - int64_t /* length */, // FIXME: this argument is not needed const int64_t* fromoffsets, int64_t offsetslength, int64_t* tooffsets) { diff --git a/awkward-cpp/src/cpu-kernels/kernel-utils.cpp b/awkward-cpp/src/cpu-kernels/kernel-utils.cpp index 22ca679629..d4b4d07000 100644 --- a/awkward-cpp/src/cpu-kernels/kernel-utils.cpp +++ b/awkward-cpp/src/cpu-kernels/kernel-utils.cpp @@ -60,7 +60,7 @@ void awkward_ListArray_combinations_step( } if (j + 1 == n) { for (int64_t k = 0; k < n; k++) { - tocarry[k][toindex[k]] = fromindex[k]; + tocarry[k][toindex[k]] = (T)fromindex[k]; toindex[k]++; } } diff --git a/dev/generate-tests.py b/dev/generate-tests.py index 99420b7939..37dc859b9a 100644 --- a/dev/generate-tests.py +++ b/dev/generate-tests.py @@ -518,7 +518,10 @@ def gencpukerneltests(specdict): ) f.write( - "import ctypes\nimport pytest\n\nfrom awkward_cpp.cpu_kernels import lib\n\n" + "import ctypes\n" + "import numpy as np\n" + "import pytest\n\n" + "from awkward_cpp.cpu_kernels import lib\n\n" ) num = 1 if spec.tests == []: @@ -547,13 +550,29 @@ def gencpukerneltests(specdict): elif count == 2: f.write( " " * 4 - + f"{arg}_ptr = (ctypes.POINTER(ctypes.c_{typename}) * len({arg}))()\n" + + f"{typename}Ptr = ctypes.POINTER(ctypes.c_{typename})\n" + " " * 4 - + f"for i in range(len({arg})):\n" + + f"{typename}PtrPtr = ctypes.POINTER({typename}Ptr)\n" + + " " * 4 + + f"dim0 = len({arg})\n" + + " " * 4 + + f"dim1 = len({arg}[0])\n" + + " " * 4 + + f"{arg}_np_arr_2d = np.empty([dim0, dim1], dtype=np.{typename})\n" + + " " * 4 + + "for i in range(dim0):\n" + " " * 8 - + f"{arg}_ptr[i] = (ctypes.c_{typename} * len({arg}[i]))(*{arg}[i])\n" + + "for j in range(dim1):\n" + + " " * 12 + + f"{arg}_np_arr_2d[i][j] = {arg}[i][j]\n" + + " " * 4 + + f"{arg}_ct_arr = np.ctypeslib.as_ctypes({arg}_np_arr_2d)\n" + + " " * 4 + + f"{typename}PtrArr = {typename}Ptr * {arg}_ct_arr._length_\n" + " " * 4 - + f"{arg} = {arg}_ptr\n" + + f"{arg}_ct_ptr = ctypes.cast({typename}PtrArr(*(ctypes.cast(row, {typename}Ptr) for row in {arg}_ct_arr)), {typename}PtrPtr)\n" + + " " * 4 + + f"{arg} = {arg}_ct_ptr\n" ) f.write(" " * 4 + "funcC = getattr(lib, '" + spec.name + "')\n") args = "" @@ -627,6 +646,7 @@ def gencpuunittests(specdict): f.write( "import ctypes\n" + "import numpy as np\n" "import pytest\n\n" "from awkward_cpp.cpu_kernels import lib\n\n" ) @@ -662,13 +682,29 @@ def gencpuunittests(specdict): elif count == 2: f.write( " " * 4 - + f"{arg}_ptr = (ctypes.POINTER(ctypes.c_{typename}) * len({arg}))()\n" + + f"{typename}Ptr = ctypes.POINTER(ctypes.c_{typename})\n" + + " " * 4 + + f"{typename}PtrPtr = ctypes.POINTER({typename}Ptr)\n" + + " " * 4 + + f"dim0 = len({arg})\n" + + " " * 4 + + f"dim1 = len({arg}[0])\n" + + " " * 4 + + f"{arg}_np_arr_2d = np.empty([dim0, dim1], dtype=np.{typename})\n" + " " * 4 - + f"for i in range(len({arg})):\n" + + "for i in range(dim0):\n" + " " * 8 - + f"{arg}_ptr[i] = (ctypes.c_{typename} * len({arg}[i]))(*{arg}[i])\n" + + "for j in range(dim1):\n" + + " " * 12 + + f"{arg}_np_arr_2d[i][j] = {arg}[i][j]\n" + " " * 4 - + f"{arg} = {arg}_ptr\n" + + f"{arg}_ct_arr = np.ctypeslib.as_ctypes({arg}_np_arr_2d)\n" + + " " * 4 + + f"{typename}PtrArr = {typename}Ptr * {arg}_ct_arr._length_\n" + + " " * 4 + + f"{arg}_ct_ptr = ctypes.cast({typename}PtrArr(*(ctypes.cast(row, {typename}Ptr) for row in {arg}_ct_arr)), {typename}PtrPtr)\n" + + " " * 4 + + f"{arg} = {arg}_ct_ptr\n" ) for arg, val in test["inputs"].items(): typename = gettype(arg, spec.args) @@ -684,13 +720,29 @@ def gencpuunittests(specdict): elif count == 2: f.write( " " * 4 - + f"{arg}_ptr = (ctypes.POINTER(ctypes.c_{typename}) * len({arg}))()\n" + + f"{typename}Ptr = ctypes.POINTER(ctypes.c_{typename})\n" + + " " * 4 + + f"{typename}PtrPtr = ctypes.POINTER({typename}Ptr)\n" + + " " * 4 + + f"dim0 = len({arg})\n" + " " * 4 - + f"for i in range(len({arg})):\n" + + f"dim1 = len({arg}[0])\n" + + " " * 4 + + f"{arg}_np_arr_2d = np.empty([dim0, dim1], dtype=np.{typename})\n" + + " " * 4 + + "for i in range(dim0):\n" + " " * 8 - + f"{arg}_ptr[i] = (ctypes.c_{typename} * len({arg}[i]))(*{arg}[i])\n" + + "for j in range(dim1):\n" + + " " * 12 + + f"{arg}_np_arr_2d[i][j] = {arg}[i][j]\n" + + " " * 4 + + f"{arg}_ct_arr = np.ctypeslib.as_ctypes({arg}_np_arr_2d)\n" + + " " * 4 + + f"{typename}PtrArr = {typename}Ptr * {arg}_ct_arr._length_\n" + + " " * 4 + + f"{arg}_ct_ptr = ctypes.cast({typename}PtrArr(*(ctypes.cast(row, {typename}Ptr) for row in {arg}_ct_arr)), {typename}PtrPtr)\n" + " " * 4 - + f"{arg} = {arg}_ptr\n" + + f"{arg} = {arg}_ct_ptr\n" ) f.write(" " * 4 + "funcC = getattr(lib, '" + spec.name + "')\n") @@ -1183,7 +1235,7 @@ def genunittests(): os.path.join(CURRENT_DIR, "..", "awkward-cpp", "tests-spec-explicit", func), "w", ) as file: - file.write("import pytest\nimport numpy\nimport kernels\n\n") + file.write("import pytest\n" "import numpy\n" "import kernels\n\n") for test in function["tests"]: num += 1 funcName = "def test_" + function["name"] + "_" + str(num) + "():\n" diff --git a/kernel-specification.yml b/kernel-specification.yml index ad3c5b5265..c58bc4d287 100644 --- a/kernel-specification.yml +++ b/kernel-specification.yml @@ -5182,14 +5182,13 @@ kernels: - name: awkward_unique_ranges_bool args: - {name: toptr, type: "List[bool]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} description: null definition: | def awkward_unique_ranges_bool( - toptr, length, fromoffsets, offsetslength, tooffsets + toptr, fromoffsets, offsetslength, tooffsets ): m = 0 for i in range(offsetslength - 1): @@ -5208,77 +5207,67 @@ kernels: - name: awkward_unique_ranges_int8 args: - {name: toptr, type: "List[int8_t]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} - name: awkward_unique_ranges_int16 args: - {name: toptr, type: "List[int16_t]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} - name: awkward_unique_ranges_int32 args: - {name: toptr, type: "List[int32_t]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} - name: awkward_unique_ranges_int64 args: - {name: toptr, type: "List[int64_t]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} - name: awkward_unique_ranges_uint8 args: - {name: toptr, type: "List[uint8_t]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} - name: awkward_unique_ranges_uint16 args: - {name: toptr, type: "List[uint16_t]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} - name: awkward_unique_ranges_uint32 args: - {name: toptr, type: "List[uint32_t]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} - name: awkward_unique_ranges_uint64 args: - {name: toptr, type: "List[uint64_t]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} - name: awkward_unique_ranges_float32 args: - {name: toptr, type: "List[float]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} - name: awkward_unique_ranges_float64 args: - {name: toptr, type: "List[double]", dir: out} - - {name: length, type: "int64_t", dir: in, role: default} - {name: fromoffsets, type: "Const[List[int64_t]]", dir: in, role: default} - {name: offsetslength, type: "int64_t", dir: in, role: default} - {name: tooffsets, type: "List[int64_t]", dir: out, role: default} description: null definition: | def awkward_unique_ranges( - toptr, length, fromoffsets, offsetslength, tooffsets + toptr, fromoffsets, offsetslength, tooffsets ): m = 0 for i in range(offsetslength - 1): diff --git a/kernel-test-data.json b/kernel-test-data.json index b3e27a323a..fde02211fa 100644 --- a/kernel-test-data.json +++ b/kernel-test-data.json @@ -26365,8 +26365,7 @@ "inputs": { "toptr": [], "fromoffsets": [0], - "offsetslength": 1, - "length": 0 + "offsetslength": 1 }, "outputs": { "toptr": [], @@ -26379,8 +26378,7 @@ "inputs": { "toptr": [1, 2], "fromoffsets": [0, 2], - "offsetslength": 2, - "length": 2 + "offsetslength": 2 }, "outputs": { "toptr": [1, 2], @@ -26393,8 +26391,7 @@ "inputs": { "toptr": [0, 1, 2, 3, 4, 5], "fromoffsets": [0, 3, 5, 6], - "offsetslength": 4, - "length": 6 + "offsetslength": 4 }, "outputs": { "toptr": [0, 1, 2, 3, 4, 5], @@ -26407,8 +26404,7 @@ "inputs": { "toptr": [3, 2, 1, 0], "fromoffsets": [0, 0, 3, 3], - "offsetslength": 4, - "length": 4 + "offsetslength": 4 }, "outputs": { "toptr": [3, 3, 1, 0], @@ -26427,8 +26423,7 @@ "inputs": { "toptr": [], "fromoffsets": [0], - "offsetslength": 1, - "length": 1 + "offsetslength": 1 }, "outputs": { "toptr": [], @@ -26441,8 +26436,7 @@ "inputs": { "toptr": [1, 1], "fromoffsets": [0, 2], - "offsetslength": 2, - "length": 1 + "offsetslength": 2 }, "outputs": { "toptr": [1, 1], @@ -26455,8 +26449,7 @@ "inputs": { "toptr": [0, 1, 1, 1, 1, 1], "fromoffsets": [0, 3, 5, 6], - "offsetslength": 4, - "length": 6 + "offsetslength": 4 }, "outputs": { "toptr": [0, 1, 1, 1, 1, 1], @@ -26469,8 +26462,7 @@ "inputs": { "toptr": [1, 1, 1, 0], "fromoffsets": [0, 0, 3, 3], - "offsetslength": 4, - "length": 4 + "offsetslength": 4 }, "outputs": { "toptr": [1, 1, 0, 0], diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index 3cd0bdf113..b76584e11e 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -831,7 +831,6 @@ def _unique(self, negaxis, starts, parents, outlength): nextoffsets.dtype.type, ]( out, - out.shape[0], offsets.data, offsets.length, nextoffsets.data, @@ -846,7 +845,6 @@ def _unique(self, negaxis, starts, parents, outlength): nextoffsets.dtype.type, ]( out, - out.shape[0], offsets.data, offsets.length, nextoffsets.data,