Skip to content

Commit

Permalink
Fix OverflowError: Python int too large to convert to C long (#2000)
Browse files Browse the repository at this point in the history
  • Loading branch information
kounelisagis authored Jul 5, 2024
1 parent f0942d9 commit 2df75c8
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 5 deletions.
6 changes: 1 addition & 5 deletions tiledb/libtiledb.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -627,11 +627,7 @@ def index_domain_subarray(array: Array, dom, idx: tuple):
if not isinstance(dim_slice, slice):
raise IndexError("invalid index type: {!r}".format(type(dim_slice)))

# numpy2 doesn't allow addition beween int and np.int64 - NEP 50
start, stop, step = dim_slice.start, dim_slice.stop, dim_slice.step
start = np.int64(start) if isinstance(start, int) else start
stop = np.int64(stop) if isinstance(stop, int) else stop
step = np.int64(step) if isinstance(step, int) else step

if np.issubdtype(dim_dtype, np.str_) or np.issubdtype(dim_dtype, np.bytes_):
if start is None or stop is None:
Expand Down Expand Up @@ -686,7 +682,7 @@ def index_domain_subarray(array: Array, dom, idx: tuple):
elif not isinstance(start, _inttypes):
raise IndexError("cannot index integral domain dimension with non-integral slice (dtype: {})".format(type(start)))
if not is_datetime and stop < 0:
stop += dim_ub
stop = np.int64(stop) + dim_ub
if stop > dim_ub:
# numpy allows stop value > than the array dimension shape,
# clamp to upper bound of dimension domain
Expand Down
32 changes: 32 additions & 0 deletions tiledb/tests/test_fixes.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,38 @@


class FixesTest(DiskTestCase):
def test_sc50378_overflowerror_python_int_too_large_to_convert_to_c_long(self):
uri = self.path(
"test_sc50378_overflowerror_python_int_too_large_to_convert_to_c_long"
)
MAX_UINT64 = np.iinfo(np.uint64).max
dim = tiledb.Dim(
name="id",
domain=(0, MAX_UINT64 - 1),
dtype=np.dtype(np.uint64),
)
dom = tiledb.Domain(dim)
text_attr = tiledb.Attr(name="text", dtype=np.dtype("U1"), var=True)
attrs = [text_attr]
schema = tiledb.ArraySchema(
domain=dom,
sparse=True,
allows_duplicates=False,
attrs=attrs,
)
tiledb.Array.create(uri, schema)

with tiledb.open(uri, "w") as A:
external_ids = np.array([0, 100, MAX_UINT64 - 1], dtype=np.dtype(np.uint64))
data = {"text": np.array(["foo", "bar", "baz"], dtype="<U3")}
A[external_ids] = data

array = tiledb.open(uri, "r", timestamp=None, config=None)
array[0]["text"][0]
array[100]["text"][0]
# This used to fail
array[MAX_UINT64 - 1]["text"][0]

def test_ch7727_float32_dim_estimate_incorrect(self):
# set max allocation: because windows won't overallocate
with tiledb.scope_ctx({"py.alloc_max_bytes": 1024**2 * 100}):
Expand Down

0 comments on commit 2df75c8

Please sign in to comment.