PyArray conversion speedups and PyArrayFromBuffer (#487)

* PyArray conversion speedups and PyArrayFromBuffer

* PyArray_Info parameterised T,N and immutable
* PyArray_Info size and stride changed to Tuples (from Vectors)
* PyArray_Info data changed from Ptr{Cvoid} to Ptr{T}
* `PyArrayInfoFromBuffer(o::PyObject)` faster way to get PyArray_Info from numpy than numpy's __array_interface__
* `PyArrayFromBuffer(o::PyObject)` 4x faster PyArray conversion
* ArrayFromBuffer no copy conversion to Julia Array
* moved PyBuffer tests to separate file

* Add isbuftype and use in pysequence_query

* Increase robustness in parsing buffer format string

* ArrayFromBuffer Improvements

* ArgumentError for non-native endian and typestrs_native

* Move PyArray related things to their own file

rename ArrayFromBuffer to NoCopyArray, and make indexing of NoCopyArray match py indexing for row major arrays too

* Void to Nothing/Cvoid

* 0.7 deprecations

* put my thang down flip it and reverse it

* Improve lifecycle management of Pybuffers

PyArray no holds a ref to the PyBuffer object. This should ensure that the buffer isn't released until the PyArray is no longer needed

setdata! now explicitly releases the buffer it used to point to

* Avoid tests when NumPy not available, and fix a test

* Clarify pydecref(o::PyBuffer) docstring

and change pass a PyPtr_NULL instead of a C_NULL for Py_buffer.obj when creating a new PyBuffer

* Remove exports: setdata!, NoCopyArray, isbuftype

* Test setdata!, improve NoCopyArray docstring

* Add tests for PyArray getindex and fix similar for PyArray

indexing into a PyArray was throwing a ambiguous method error for similar(::PyArray, T, dims::Dims)

* Add GC rooting note

benchmarks/arrayperf.jl

@@ -0,0 +1,57 @@
+using PyCall, BenchmarkTools, DataStructures
+using PyCall: PyArray_Info
+
+results = OrderedDict{String,Any}()
+
+let
+    np = pyimport("numpy")
+    nprand = np["random"]["rand"]
+    # nparray_pyo(x) = pycall(np["array"], PyObject, x)
+    # pytestarray(sz::Int...) = pycall(np["reshape"], PyObject, nparray_pyo(1:prod(sz)), sz)
+
+    # no convert baseline
+    nprand_pyo(sz...)   = pycall(nprand, PyObject, sz...)
+
+    for arr_size in [(2,2), (100,100)]
+        pyo_arr = nprand_pyo(arr_size...)
+        results["nprand_pyo$arr_size"] = @benchmark $nprand_pyo($arr_size...)
+        println("nprand_pyo $arr_size:\n"); display(results["nprand_pyo$arr_size"])
+        println("--------------------------------------------------")
+
+        results["convert_pyarr$arr_size"] = @benchmark $convert(PyArray, $pyo_arr)
+        println("convert_pyarr $arr_size:\n"); display(results["convert_pyarr$arr_size"])
+        println("--------------------------------------------------")
+
+        results["PyArray_Info$arr_size"] = @benchmark $PyArray_Info($pyo_arr)
+        println("PyArray_Info $arr_size:\n"); display(results["PyArray_Info$arr_size"])
+        println("--------------------------------------------------")
+
+        results["convert_pyarrbuf$arr_size"] = @benchmark $PyArray($pyo_arr)
+        println("convert_pyarrbuf $arr_size:\n"); display(results["convert_pyarrbuf$arr_size"])
+        println("--------------------------------------------------")
+
+        results["convert_arr$arr_size"] = @benchmark convert(Array, $pyo_arr)
+        println("convert_arr $arr_size:\n"); display(results["convert_arr$arr_size"])
+        println("--------------------------------------------------")
+
+        results["convert_arrbuf$arr_size"] = @benchmark $NoCopyArray($pyo_arr)
+        println("convert_arrbuf $arr_size:\n"); display(results["convert_arrbuf$arr_size"])
+        println("--------------------------------------------------")
+
+        pyarr = convert(PyArray, pyo_arr)
+        results["setdata!$arr_size"] = @benchmark $setdata!($pyarr, $pyo_arr)
+        println("setdata!:\n"); display(results["setdata!$arr_size"])
+        println("--------------------------------------------------")
+
+        pyarr = convert(PyArray, pyo_arr)
+        pybuf=PyBuffer()
+        results["setdata! bufprealloc$arr_size"] =
+            @benchmark $setdata!($pyarr, $pyo_arr, $pybuf)
+        println("setdata! bufprealloc:\n"); display(results["setdata! bufprealloc$arr_size"])
+        println("--------------------------------------------------")
+    end
+end
+println()
+println("Mean times")
+println("----------")
+foreach((r)->println(rpad(r[1],27), ": ", mean(r[2])), results)

src/PyCall.jl

@@ -5,7 +5,8 @@ module PyCall
 using Compat, VersionParsing
 
 export pycall, pycall!, pyimport, pyimport_e, pybuiltin, PyObject, PyReverseDims,
-       PyPtr, pyincref, pydecref, pyversion, PyArray, PyArray_Info,
+       PyPtr, pyincref, pydecref, pyversion,
+       PyArray, PyArray_Info, PyBuffer,
        pyerr_check, pyerr_clear, pytype_query, PyAny, @pyimport, PyDict,
        pyisinstance, pywrap, pytypeof, pyeval, PyVector, pystring, pystr, pyrepr,
        pyraise, pytype_mapping, pygui, pygui_start, pygui_stop,
@@ -177,6 +178,7 @@ pytypeof(o::PyObject) = ispynull(o) ? throw(ArgumentError("NULL PyObjects have n
 
 const TypeTuple = Union{Type,NTuple{N, Type}} where {N}
 include("pybuffer.jl")
+include("pyarray.jl")
 include("conversions.jl")
 include("pytype.jl")
 include("pyiterator.jl")

src/conversions.jl

@@ -768,13 +768,11 @@ function pysequence_query(o::PyObject)
         return AbstractRange
     elseif ispybytearray(o)
         return Vector{UInt8}
-    elseif !haskey(o, "__array_interface__")
+    elseif !isbuftype(o)
         # only handle PyList for now
         return pyisinstance(o, @pyglobalobj :PyList_Type) ? Array : Union{}
     else
-        otypestr = get(o["__array_interface__"], PyObject, "typestr")
-        typestr = convert(AbstractString, otypestr) # Could this just be String now?
-        T = npy_typestrs[typestr[2:end]]
+        T, native_byteorder = array_format(o)
         if T == PyPtr
             T = PyObject
         end