Default ctx initialization improvements, and custom buffer size

misc/azure-ci.yml

@@ -1,7 +1,7 @@
   #  template: ./misc/azure-ci.yml
 
 jobs:
-- job: osx
+- job:
   pool:
     vmImage: $(imageName)
   condition: not(or(startsWith(variables['Build.SourceBranch'], 'refs/tags'), contains(variables['Build.SourceBranchName'], 'release-')))

tiledb/highlevel.py

@@ -73,10 +73,12 @@ def empty_like(uri, arr, config=None, key=None, tile=None):
     return tiledb.DenseArray(uri, mode='w', key=key, ctx=ctx)
 
 
-def from_numpy(uri, array, ctx=default_ctx(), **kw):
+def from_numpy(uri, array, ctx=None, **kw):
     """
     Convenience method, see `tiledb.DenseArray.from_numpy`
     """
+    if not ctx:
+        ctx = default_ctx()
     if not isinstance(array, np.ndarray):
         raise Exception("from_numpy is only currently supported for numpy.ndarray")
 

tiledb/indexing.pyx

@@ -110,14 +110,14 @@ cdef class QueryAttr(object):
         self.name = name
         self.dtype = dtype
 
-cdef dict execute_multi_index(tiledb_ctx_t* ctx_ptr,
-                        tiledb_query_t* query_ptr,
-                        Array array,
-                        tuple attr_names,
-                        return_coord):
+cdef dict execute_multi_index(Array array,
+                              tiledb_query_t* query_ptr,
+                              tuple attr_names,
+                              return_coord):
 
     # NOTE: query_ptr *must* only be freed in caller
 
+    cdef tiledb_ctx_t* ctx_ptr = array.ctx.ptr
     cdef np.dtype coords_dtype
     cdef unicode coord_name = (tiledb_coords()).decode('UTF-8')
     cdef uint64_t attr_idx
@@ -132,7 +132,6 @@ cdef dict execute_multi_index(tiledb_ctx_t* ctx_ptr,
     cdef uint64_t el_count
     cdef QueryAttr qattr
 
-    cdef void* attr_array_ptr = NULL
     cdef uint64_t* buffer_sizes_ptr = NULL
 
     cdef bint repeat_query = True
@@ -158,7 +157,9 @@ cdef dict execute_multi_index(tiledb_ctx_t* ctx_ptr,
     cdef np.ndarray buffer_sizes = np.zeros(nattr, np.uint64)
     cdef np.ndarray result_bytes_read = np.zeros(nattr, np.uint64)
 
-    cdef uint64_t init_element_count = 1310720 # 10 MB int64
+    cdef uint64_t init_buffer_size = 1310720 * 8 # 10 MB int64
+    if 'py.init_buffer_bytes' in array.ctx.config():
+        init_buffer_size = int(array.ctx.config()['py.init_buffer_bytes'])
     # switch from exponential to linear (+4GB) allocation
     cdef uint64_t linear_alloc_bytes = 4 * (2**30) # 4 GB
 
@@ -178,8 +179,9 @@ cdef dict execute_multi_index(tiledb_ctx_t* ctx_ptr,
             attr_name = qattr.name
             attr_dtype = qattr.dtype
 
+            # allocate initial array
             if repeat_count == 0:
-                result_dict[attr_name] = np.zeros(init_element_count,
+                result_dict[attr_name] = np.zeros(int(init_buffer_size / attr_dtype.itemsize),
                                                   dtype=attr_dtype)
 
             # Get the array here in order to save a lookup
@@ -338,7 +340,7 @@ cpdef multi_index(Array array, tuple attr_names, tuple ranges,
     try:
         if coords is None:
             coords = array.schema.sparse
-        result = execute_multi_index(ctx_ptr, query_ptr, array, attr_names, coords)
+        result = execute_multi_index(array, query_ptr, attr_names, coords)
     finally:
         tiledb_query_free(&query_ptr)