diff --git a/pysrc/juliacall/ipython.py b/pysrc/juliacall/ipython.py index 07d3d930..b416aa98 100644 --- a/pysrc/juliacall/ipython.py +++ b/pysrc/juliacall/ipython.py @@ -69,12 +69,12 @@ def load_ipython_extension(ip): # redirect stdout/stderr if ip.__class__.__name__ == 'TerminalInteractiveShell': # no redirection in the terminal - PythonCall.seval("""begin + PythonCall.seval("""module _ipython function _flush_stdio() end end""") else: - PythonCall.seval("""begin + PythonCall.seval("""module _ipython const _redirected_stdout = redirect_stdout() const _redirected_stderr = redirect_stderr() const _py_stdout = PyIO(pyimport("sys" => "stdout"); line_buffering=true) @@ -92,10 +92,10 @@ def load_ipython_extension(ip): end nothing end""") - ip.events.register('post_execute', PythonCall._flush_stdio) + ip.events.register('post_execute', PythonCall._ipython._flush_stdio) # push displays PythonCall.seval("""begin - pushdisplay(PythonDisplay()) - pushdisplay(IPythonDisplay()) + pushdisplay(_compat.PythonDisplay()) + pushdisplay(_compat.IPythonDisplay()) nothing end""") diff --git a/src/CPython/_.jl b/src/CPython/_.jl new file mode 100644 index 00000000..c2be2d13 --- /dev/null +++ b/src/CPython/_.jl @@ -0,0 +1,30 @@ +""" + module _CPython + +This module provides a direct interface to the Python C API. +""" +module _CPython + +using Base: @kwdef +using UnsafePointers: UnsafePtr +using CondaPkg: CondaPkg +using Pkg: Pkg +using Requires: @require +using Libdl: dlpath, dlopen, dlopen_e, dlclose, dlsym, dlsym_e, RTLD_LAZY, RTLD_DEEPBIND, RTLD_GLOBAL + +# import Base: @kwdef +# import CondaPkg +# import Pkg +# using Libdl, Requires, UnsafePointers, Serialization, ..Utils + +include("consts.jl") +include("pointers.jl") +include("extras.jl") +include("context.jl") +include("gil.jl") + +function __init__() + init_context() +end + +end diff --git a/src/cpython/consts.jl b/src/CPython/consts.jl similarity index 98% rename from src/cpython/consts.jl rename to src/CPython/consts.jl index b4047d7a..72e49d34 100644 --- a/src/cpython/consts.jl +++ b/src/CPython/consts.jl @@ -324,12 +324,6 @@ const PyTypePtr = Ptr{PyTypeObject} value::T end -@kwdef struct PyJuliaValueObject - ob_base::PyObject = PyObject() - value::Int = 0 - weaklist::PyPtr = C_NULL -end - @kwdef struct PyArrayInterface two::Cint = 0 nd::Cint = 0 diff --git a/src/cpython/context.jl b/src/CPython/context.jl similarity index 98% rename from src/cpython/context.jl rename to src/CPython/context.jl index 3ddb1248..30281259 100644 --- a/src/cpython/context.jl +++ b/src/CPython/context.jl @@ -137,11 +137,14 @@ function init_context() # Get function pointers from the library init_pointers() + # Compare libpath with PyCall + @require PyCall="438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall) + # Initialize the interpreter with_gil() do CTX.is_preinitialized = Py_IsInitialized() != 0 if CTX.is_preinitialized - @assert CTX.which == :PyCall + @assert CTX.which == :PyCall || CTX.matches_pycall isa Bool else @assert CTX.which != :PyCall # Find ProgramName and PythonHome @@ -211,9 +214,6 @@ function init_context() ENV["JULIA_PYTHONCALL_EXE"] = CTX.exe_path::String end - # Compare libpath with PyCall - @require PyCall="438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall) - with_gil() do # Get the python version diff --git a/src/cpython/extras.jl b/src/CPython/extras.jl similarity index 100% rename from src/cpython/extras.jl rename to src/CPython/extras.jl diff --git a/src/cpython/find_libpython.py b/src/CPython/find_libpython.py similarity index 100% rename from src/cpython/find_libpython.py rename to src/CPython/find_libpython.py diff --git a/src/cpython/gil.jl b/src/CPython/gil.jl similarity index 100% rename from src/cpython/gil.jl rename to src/CPython/gil.jl diff --git a/src/cpython/pointers.jl b/src/CPython/pointers.jl similarity index 98% rename from src/cpython/pointers.jl rename to src/CPython/pointers.jl index 452eb34c..4d63c578 100644 --- a/src/cpython/pointers.jl +++ b/src/CPython/pointers.jl @@ -274,8 +274,6 @@ const CAPI_OBJECTS = Set([ $([:($name :: PyPtr = C_NULL) for name in CAPI_EXCEPTIONS]...) $([:($name :: PyPtr = C_NULL) for name in CAPI_OBJECTS]...) PyOS_InputHookPtr :: Ptr{Ptr{Cvoid}} = C_NULL - PyJuliaBase_Type :: PyPtr = C_NULL - PyExc_JuliaError :: PyPtr = C_NULL end const POINTERS = CAPIPointers() @@ -289,7 +287,7 @@ const POINTERS = CAPIPointers() end for name in CAPI_FUNCS ]...) - $([:(p.$name = Base.unsafe_load(Ptr{PyPtr}(dlsym(lib, $(QuoteNode(name)))))) for name in CAPI_EXCEPTIONS]...) + $([:(p.$name = Base.unsafe_load(Ptr{PyPtr}(dlsym(lib, $(QuoteNode(name)))::Ptr))) for name in CAPI_EXCEPTIONS]...) $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...) p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook) end diff --git a/src/Py.jl b/src/Py/Py.jl similarity index 97% rename from src/Py.jl rename to src/Py/Py.jl index 13fbdb02..754430ce 100644 --- a/src/Py.jl +++ b/src/Py/Py.jl @@ -47,6 +47,7 @@ py_finalizer(x::Py) = GC.enqueue(getptr(x)) ispy(::Py) = true getptr(x::Py) = getfield(x, :ptr) +pyconvert(::Type{Py}, x::Py) = x setptr!(x::Py, ptr::C.PyPtr) = (setfield!(x, :ptr, ptr); x) @@ -148,7 +149,6 @@ Py(x::AbstractRange{<:Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UI Py(x::Date) = pydate(x) Py(x::Time) = pytime(x) Py(x::DateTime) = pydatetime(x) -Py(x) = ispy(x) ? throw(MethodError(Py, (x,))) : pyjl(x) Base.string(x::Py) = pyisnull(x) ? "" : pystr(String, x) Base.print(io::IO, x::Py) = print(io, string(x)) @@ -251,12 +251,8 @@ function Base.show(io::IO, ::MIME"text/plain", o::Py) end end -Base.show(io::IO, mime::MIME, o::Py) = pyshow(io, mime, o) -Base.show(io::IO, mime::MIME"text/csv", o::Py) = pyshow(io, mime, o) -Base.show(io::IO, mime::MIME"text/tab-separated-values", o::Py) = pyshow(io, mime, o) Base.showable(::MIME"text/plain", ::Py) = true -Base.showable(mime::MIME, o::Py) = pyshowable(mime, o) Base.getproperty(x::Py, k::Symbol) = pygetattr(x, string(k)) Base.getproperty(x::Py, k::String) = pygetattr(x, k) diff --git a/src/Py/_.jl b/src/Py/_.jl new file mode 100644 index 00000000..0633ccd8 --- /dev/null +++ b/src/Py/_.jl @@ -0,0 +1,38 @@ +""" + module _Py + +Defines the `Py` type and directly related functions. +""" +module _Py + +const VERSION = v"0.9.15" +const ROOT_DIR = dirname(dirname(@__DIR__)) + +using .._CPython: _CPython as C +using .._Utils: _Utils as Utils +using Base: @propagate_inbounds, @kwdef +using Dates: Date, Time, DateTime, year, month, day, hour, minute, second, millisecond, microsecond, nanosecond +using MacroTools: @capture +using Markdown: Markdown +# using MacroTools, Dates, Tables, Markdown, Serialization, Requires, Pkg, REPL + +include("gc.jl") +include("Py.jl") +include("err.jl") +include("config.jl") +include("consts.jl") +include("builtins.jl") +include("stdlib.jl") +include("juliacall.jl") +include("pyconst_macro.jl") + +function __init__() + C.with_gil() do + init_consts() + init_datetime() + init_stdlib() + init_juliacall() + end +end + +end diff --git a/src/Py/builtins.jl b/src/Py/builtins.jl new file mode 100644 index 00000000..23e79693 --- /dev/null +++ b/src/Py/builtins.jl @@ -0,0 +1,1519 @@ +### object interface + +""" + pyis(x, y) + +True if `x` and `y` are the same Python object. Equivalent to `x is y` in Python. +""" +pyis(x, y) = @autopy x y getptr(x_) == getptr(y_) +export pyis + +pyisnot(x, y) = !pyis(x, y) + +""" + pyrepr(x) + +Equivalent to `repr(x)` in Python. +""" +pyrepr(x) = pynew(errcheck(@autopy x C.PyObject_Repr(getptr(x_)))) +pyrepr(::Type{String}, x) = (s=pyrepr(x); ans=pystr_asstring(s); pydel!(s); ans) +export pyrepr + +""" + pyascii(x) + +Equivalent to `ascii(x)` in Python. +""" +pyascii(x) = pynew(errcheck(@autopy x C.PyObject_ASCII(getptr(x_)))) +pyascii(::Type{String}, x) = (s=pyascii(x); ans=pystr_asstring(s); pydel!(s); ans) +export pyascii + +""" + pyhasattr(x, k) + +Equivalent to `hasattr(x, k)` in Python. + +Tests if `getattr(x, k)` raises an `AttributeError`. +""" +function pyhasattr(x, k) + ptr = @autopy x k C.PyObject_GetAttr(getptr(x_), getptr(k_)) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return false + else + pythrow() + end + else + decref(ptr) + return true + end +end +# pyhasattr(x, k) = errcheck(@autopy x k C.PyObject_HasAttr(getptr(x_), getptr(k_))) == 1 +export pyhasattr + +""" + pygetattr(x, k, [d]) + +Equivalent to `getattr(x, k)` or `x.k` in Python. + +If `d` is specified, it is returned if the attribute does not exist. +""" +pygetattr(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetAttr(getptr(x_), getptr(k_)))) +function pygetattr(x, k, d) + ptr = @autopy x k C.PyObject_GetAttr(getptr(x_), getptr(k_)) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return d + else + pythrow() + end + else + return pynew(ptr) + end +end +export pygetattr + +""" + pysetattr(x, k, v) + +Equivalent to `setattr(x, k, v)` or `x.k = v` in Python. +""" +pysetattr(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetAttr(getptr(x_), getptr(k_), getptr(v_))); nothing) +export pysetattr + +""" + pydelattr(x, k) + +Equivalent to `delattr(x, k)` or `del x.k` in Python. +""" +pydelattr(x, k) = (errcheck(@autopy x k C.PyObject_SetAttr(getptr(x_), getptr(k_), C.PyNULL)); nothing) +export pydelattr + +""" + pyissubclass(s, t) + +Test if `s` is a subclass of `t`. Equivalent to `issubclass(s, t)` in Python. +""" +pyissubclass(s, t) = errcheck(@autopy s t C.PyObject_IsSubclass(getptr(s_), getptr(t_))) == 1 +export pyissubclass + +""" + pyisinstance(x, t) + +Test if `x` is of type `t`. Equivalent to `isinstance(x, t)` in Python. +""" +pyisinstance(x, t) = errcheck(@autopy x t C.PyObject_IsInstance(getptr(x_), getptr(t_))) == 1 +export pyisinstance + +""" + pyhash(x) + +Equivalent to `hash(x)` in Python, converted to an `Integer`. +""" +pyhash(x) = errcheck(@autopy x C.PyObject_Hash(getptr(x_))) +export pyhash + +""" + pytruth(x) + +The truthyness of `x`. Equivalent to `bool(x)` in Python, converted to a `Bool`. +""" +pytruth(x) = errcheck(@autopy x C.PyObject_IsTrue(getptr(x_))) == 1 +export pytruth + +""" + pynot(x) + +The falsyness of `x`. Equivalent to `not x` in Python, converted to a `Bool`. +""" +pynot(x) = errcheck(@autopy x C.PyObject_Not(getptr(x_))) == 1 +export pynot + +""" + pylen(x) + +The length of `x`. Equivalent to `len(x)` in Python, converted to an `Integer`. +""" +pylen(x) = errcheck(@autopy x C.PyObject_Length(getptr(x_))) +export pylen + +""" + pyhasitem(x, k) + +Test if `pygetitem(x, k)` raises a `KeyError` or `AttributeError`. +""" +function pyhasitem(x, k) + ptr = @autopy x k C.PyObject_GetItem(getptr(x_), getptr(k_)) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return false + else + pythrow() + end + else + decref(ptr) + return true + end +end +export pyhasitem + +""" + pygetitem(x, k, [d]) + +Equivalent `x[k]` in Python. + +If `d` is specified, it is returned if the item does not exist (i.e. if `x[k]` raises a +`KeyError` or `IndexError`). +""" +pygetitem(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetItem(getptr(x_), getptr(k_)))) +function pygetitem(x, k, d) + ptr = @autopy x k C.PyObject_GetItem(getptr(x_), getptr(k_)) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return d + else + pythrow() + end + else + return pynew(ptr) + end +end +export pygetitem + +""" + pysetitem(x, k, v) + +Equivalent to `setitem(x, k, v)` or `x[k] = v` in Python. +""" +pysetitem(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetItem(getptr(x_), getptr(k_), getptr(v_))); nothing) +export pysetitem + +""" + pydelitem(x, k) + +Equivalent to `delitem(x, k)` or `del x[k]` in Python. +""" +pydelitem(x, k) = (errcheck(@autopy x k C.PyObject_DelItem(getptr(x_), getptr(k_))); nothing) +export pydelitem + +""" + pydir(x) + +Equivalent to `dir(x)` in Python. +""" +pydir(x) = pynew(errcheck(@autopy x C.PyObject_Dir(getptr(x_)))) +export pydir + +pycallargs(f) = pynew(errcheck(@autopy f C.PyObject_CallObject(getptr(f_), C.PyNULL))) +pycallargs(f, args) = pynew(errcheck(@autopy f args C.PyObject_CallObject(getptr(f_), getptr(args_)))) +pycallargs(f, args, kwargs) = pynew(errcheck(@autopy f args kwargs C.PyObject_Call(getptr(f_), getptr(args_), getptr(kwargs_)))) + +""" + pycall(f, args...; kwargs...) + +Call the Python object `f` with the given arguments. +""" +pycall(f, args...; kwargs...) = + if !isempty(kwargs) + args_ = pytuple_fromiter(args) + kwargs_ = pystrdict_fromiter(kwargs) + ans = pycallargs(f, args_, kwargs_) + pydel!(args_) + pydel!(kwargs_) + ans + elseif !isempty(args) + args_ = pytuple_fromiter(args) + ans = pycallargs(f, args_) + pydel!(args_) + ans + else + pycallargs(f) + end +export pycall + +""" + pyeq(x, y) + pyeq(Bool, x, y) + +Equivalent to `x == y` in Python. The second form converts to `Bool`. +""" +pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_EQ))) + +""" + pyne(x, y) + pyne(Bool, x, y) + +Equivalent to `x != y` in Python. The second form converts to `Bool`. +""" +pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_NE))) + +""" + pyle(x, y) + pyle(Bool, x, y) + +Equivalent to `x <= y` in Python. The second form converts to `Bool`. +""" +pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_LE))) + +""" + pylt(x, y) + pylt(Bool, x, y) + +Equivalent to `x < y` in Python. The second form converts to `Bool`. +""" +pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_LT))) + +""" + pyge(x, y) + pyge(Bool, x, y) + +Equivalent to `x >= y` in Python. The second form converts to `Bool`. +""" +pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_GE))) + +""" + pygt(x, y) + pygt(Bool, x, y) + +Equivalent to `x > y` in Python. The second form converts to `Bool`. +""" +pygt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_GT))) +pyeq(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_EQ)) == 1 +pyne(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_NE)) == 1 +pyle(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_LE)) == 1 +pylt(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_LT)) == 1 +pyge(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_GE)) == 1 +pygt(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_GT)) == 1 +export pyeq, pyne, pyle, pylt, pyge, pygt + +""" + pycontains(x, v) + +Equivalent to `v in x` in Python. +""" +pycontains(x, v) = errcheck(@autopy x v C.PySequence_Contains(getptr(x_), getptr(v_))) == 1 +export pycontains + +""" + pyin(v, x) + +Equivalent to `v in x` in Python. +""" +pyin(v, x) = pycontains(x, v) +export pyin + +pynotin(v, x) = !pyin(v, x) + +### number interface + +# unary +""" + pyneg(x) + +Equivalent to `-x` in Python. +""" +pyneg(x) = pynew(errcheck(@autopy x C.PyNumber_Negative(getptr(x_)))) +""" + pypos(x) + +Equivalent to `+x` in Python. +""" +pypos(x) = pynew(errcheck(@autopy x C.PyNumber_Positive(getptr(x_)))) +""" + pyabs(x) + +Equivalent to `abs(x)` in Python. +""" +pyabs(x) = pynew(errcheck(@autopy x C.PyNumber_Absolute(getptr(x_)))) +""" + pyinv(x) + +Equivalent to `~x` in Python. +""" +pyinv(x) = pynew(errcheck(@autopy x C.PyNumber_Invert(getptr(x_)))) +""" + pyindex(x) + +Convert `x` losslessly to an `int`. +""" +pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(getptr(x_)))) +export pyneg, pypos, pyabs, pyinv, pyindex + +# binary +""" + pyadd(x, y) + +Equivalent to `x + y` in Python. +""" +pyadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Add(getptr(x_), getptr(y_)))) +""" + pysub(x, y) + +Equivalent to `x - y` in Python. +""" +pysub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Subtract(getptr(x_), getptr(y_)))) +""" + pymul(x, y) + +Equivalent to `x * y` in Python. +""" +pymul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Multiply(getptr(x_), getptr(y_)))) +""" + pymatmul(x, y) + +Equivalent to `x @ y` in Python. +""" +pymatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(getptr(x_), getptr(y_)))) +""" + pyfloordiv(x, y) + +Equivalent to `x // y` in Python. +""" +pyfloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(getptr(x_), getptr(y_)))) +""" + pytruediv(x, y) + +Equivalent to `x / y` in Python. +""" +pytruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(getptr(x_), getptr(y_)))) +""" + pymod(x, y) + +Equivalent to `x % y` in Python. +""" +pymod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Remainder(getptr(x_), getptr(y_)))) +""" + pydivmod(x, y) + +Equivalent to `divmod(x, y)` in Python. +""" +pydivmod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Divmod(getptr(x_), getptr(y_)))) +""" + pylshift(x, y) + +Equivalent to `x << y` in Python. +""" +pylshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Lshift(getptr(x_), getptr(y_)))) +""" + pyrshift(x, y) + +Equivalent to `x >> y` in Python. +""" +pyrshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Rshift(getptr(x_), getptr(y_)))) +""" + pyand(x, y) + +Equivalent to `x & y` in Python. +""" +pyand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_And(getptr(x_), getptr(y_)))) +""" + pyxor(x, y) + +Equivalent to `x ^ y` in Python. +""" +pyxor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Xor(getptr(x_), getptr(y_)))) +""" + pyor(x, y) + +Equivalent to `x | y` in Python. +""" +pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(getptr(x_), getptr(y_)))) +export pyadd, pysub, pymul, pymatmul, pyfloordiv, pytruediv, pymod, pydivmod, pylshift, pyrshift, pyand, pyxor, pyor + +# binary in-place +""" + pyiadd(x, y) + +In-place add. `x = pyiadd(x, y)` is equivalent to `x += y` in Python. +""" +pyiadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(getptr(x_), getptr(y_)))) +""" + pyisub(x, y) + +In-place subtract. `x = pyisub(x, y)` is equivalent to `x -= y` in Python. +""" +pyisub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(getptr(x_), getptr(y_)))) +""" + pyimul(x, y) + +In-place multiply. `x = pyimul(x, y)` is equivalent to `x *= y` in Python. +""" +pyimul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(getptr(x_), getptr(y_)))) +""" + pyimatmul(x, y) + +In-place matrix multiply. `x = pyimatmul(x, y)` is equivalent to `x @= y` in Python. +""" +pyimatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(getptr(x_), getptr(y_)))) +""" + pyifloordiv(x, y) + +In-place floor divide. `x = pyifloordiv(x, y)` is equivalent to `x //= y` in Python. +""" +pyifloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(getptr(x_), getptr(y_)))) +""" + pyitruediv(x, y) + +In-place true division. `x = pyitruediv(x, y)` is equivalent to `x /= y` in Python. +""" +pyitruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(getptr(x_), getptr(y_)))) +""" + pyimod(x, y) + +In-place subtraction. `x = pyimod(x, y)` is equivalent to `x %= y` in Python. +""" +pyimod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(getptr(x_), getptr(y_)))) +""" + pyilshift(x, y) + +In-place left shift. `x = pyilshift(x, y)` is equivalent to `x <<= y` in Python. +""" +pyilshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(getptr(x_), getptr(y_)))) +""" + pyirshift(x, y) + +In-place right shift. `x = pyirshift(x, y)` is equivalent to `x >>= y` in Python. +""" +pyirshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(getptr(x_), getptr(y_)))) +""" + pyiand(x, y) + +In-place and. `x = pyiand(x, y)` is equivalent to `x &= y` in Python. +""" +pyiand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(getptr(x_), getptr(y_)))) +""" + pyixor(x, y) + +In-place xor. `x = pyixor(x, y)` is equivalent to `x ^= y` in Python. +""" +pyixor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(getptr(x_), getptr(y_)))) +""" + pyior(x, y) + +In-place or. `x = pyior(x, y)` is equivalent to `x |= y` in Python. +""" +pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(getptr(x_), getptr(y_)))) +export pyiadd, pyisub, pyimul, pyimatmul, pyifloordiv, pyitruediv, pyimod, pyilshift, pyirshift, pyiand, pyixor, pyior + +# power +""" + pypow(x, y, z=None) + +Equivalent to `x ** y` or `pow(x, y, z)` in Python. +""" +pypow(x, y, z=pybuiltins.None) = pynew(errcheck(@autopy x y z C.PyNumber_Power(getptr(x_), getptr(y_), getptr(z_)))) +""" + pyipow(x, y, z=None) + +In-place power. `x = pyipow(x, y)` is equivalent to `x **= y` in Python. +""" +pyipow(x, y, z=pybuiltins.None) = pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(getptr(x_), getptr(y_), getptr(z_)))) +export pypow, pyipow + +### iter + +""" + pyiter(x) + +Equivalent to `iter(x)` in Python. +""" +pyiter(x) = pynew(errcheck(@autopy x C.PyObject_GetIter(getptr(x_)))) +export pyiter + +""" + pynext(x) + +Equivalent to `next(x)` in Python. +""" +pynext(x) = pybuiltins.next(x) +export pynext + +""" + unsafe_pynext(x) + +Return the next item in the iterator `x`. When there are no more items, return NULL. +""" +unsafe_pynext(x::Py) = Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(getptr(x)))) + +### None + +pyisnone(x) = pyis(x, pybuiltins.None) + +### bool + +""" + pybool(x) + +Convert `x` to a Python `bool`. +""" +pybool(x::Bool=false) = pynew(x ? pybuiltins.True : pybuiltins.False) +pybool(x::Number) = pybool(!iszero(x)) +pybool(x) = pybuiltins.bool(x) +export pybool + +pyisTrue(x) = pyis(x, pybuiltins.True) +pyisFalse(x) = pyis(x, pybuiltins.False) +pyisbool(x) = pyisTrue(x) || pyisFalse(x) + +function pybool_asbool(x) + @autopy x if pyisTrue(x_) + true + elseif pyisFalse(x_) + false + else + error("not a bool") + end +end + +### str + +pystr_fromUTF8(x::Ptr, n::Integer) = pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) +pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) + +""" + pystr(x) + +Convert `x` to a Python `str`. +""" +pystr(x) = pynew(errcheck(@autopy x C.PyObject_Str(getptr(x_)))) +pystr(x::String) = pystr_fromUTF8(x) +pystr(x::SubString{String}) = pystr_fromUTF8(x) +pystr(x::Char) = pystr(string(x)) +pystr(::Type{String}, x) = (s=pystr(x); ans=pystr_asstring(s); pydel!(s); ans) +export pystr + +pystr_asUTF8bytes(x::Py) = Base.GC.@preserve x pynew(errcheck(C.PyUnicode_AsUTF8String(getptr(x)))) +pystr_asUTF8vector(x::Py) = (b=pystr_asUTF8bytes(x); ans=pybytes_asvector(b); pydel!(b); ans) +pystr_asstring(x::Py) = (b=pystr_asUTF8bytes(x); ans=pybytes_asUTF8string(b); pydel!(b); ans) + +function pystr_intern!(x::Py) + ptr = Ref(getptr(x)) + C.PyUnicode_InternInPlace(ptr) + setptr!(x, ptr[]) +end + +pyisstr(x) = pytypecheckfast(x, C.Py_TPFLAGS_UNICODE_SUBCLASS) + +### bytes + +pybytes_fromdata(x::Ptr, n::Integer) = pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) +pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) + +""" + pybytes(x) + +Convert `x` to a Python `bytes`. +""" +pybytes(x) = pynew(errcheck(@autopy x C.PyObject_Bytes(getptr(x_)))) +pybytes(x::Vector{UInt8}) = pybytes_fromdata(x) +pybytes(x::Base.CodeUnits{UInt8, String}) = pybytes_fromdata(x) +pybytes(x::Base.CodeUnits{UInt8, SubString{String}}) = pybytes_fromdata(x) +pybytes(::Type{T}, x) where {Vector{UInt8} <: T <: Vector} = (b=pybytes(x); ans=pybytes_asvector(b); pydel!(b); ans) +pybytes(::Type{T}, x) where {Base.CodeUnits{UInt8,String} <: T <: Base.CodeUnits} = (b=pybytes(x); ans=Base.CodeUnits(pybytes_asUTF8string(b)); pydel!(b); ans) +export pybytes + +pyisbytes(x) = pytypecheckfast(x, C.Py_TPFLAGS_BYTES_SUBCLASS) + +function pybytes_asdata(x::Py) + ptr = Ref(Ptr{Cchar}(0)) + len = Ref(C.Py_ssize_t(0)) + Base.GC.@preserve x errcheck(C.PyBytes_AsStringAndSize(getptr(x), ptr, len)) + ptr[], len[] +end + +function pybytes_asvector(x::Py) + ptr, len = pybytes_asdata(x) + unsafe_wrap(Array, Ptr{UInt8}(ptr), len) +end + +function pybytes_asUTF8string(x::Py) + ptr, len = pybytes_asdata(x) + unsafe_string(Ptr{UInt8}(ptr), len) +end + +### int + +pyint_fallback(x::Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}) = + pynew(errcheck(C.PyLong_FromString(string(x, base=32), C_NULL, 32))) +pyint_fallback(x::Integer) = pyint_fallback(BigInt(x)) + +""" + pyint(x=0) + +Convert `x` to a Python `int`. +""" +function pyint(x::Integer=0) + y = mod(x, Clonglong) + if x == y + pynew(errcheck(C.PyLong_FromLongLong(y))) + else + pyint_fallback(x) + end +end +function pyint(x::Unsigned) + y = mod(x, Culonglong) + if x == y + pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) + else + pyint_fallback(x) + end +end +pyint(x) = @autopy x pynew(errcheck(C.PyNumber_Long(getptr(x_)))) +export pyint + +pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) + +### float + +""" + pyfloat(x=0.0) + +Convert `x` to a Python `float`. +""" +pyfloat(x::Real=0.0) = pynew(errcheck(C.PyFloat_FromDouble(x))) +pyfloat(x) = @autopy x pynew(errcheck(C.PyNumber_Float(getptr(x_)))) +export pyfloat + +pyisfloat(x) = pytypecheck(x, pybuiltins.float) + +pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(getptr(x_))) + +### complex + +""" + pycomplex(x=0.0) + pycomplex(re, im) + +Convert `x` to a Python `complex`, or create one from given real and imaginary parts. +""" +pycomplex(x::Real=0.0, y::Real=0.0) = pynew(errcheck(C.PyComplex_FromDoubles(x, y))) +pycomplex(x::Complex) = pycomplex(real(x), imag(x)) +pycomplex(x) = pybuiltins.complex(x) +pycomplex(x, y) = pybuiltins.complex(x, y) +export pycomplex + +pyiscomplex(x) = pytypecheck(x, pybuiltins.complex) + +function pycomplex_ascomplex(x) + c = @autopy x C.PyComplex_AsCComplex(getptr(x_)) + c.real == -1 && c.imag == 0 && errcheck() + return Complex(c.real, c.imag) +end + +### type + +""" + pytype(x) + +The Python `type` of `x`. +""" +pytype(x) = pynew(errcheck(@autopy x C.PyObject_Type(getptr(x_)))) +export pytype + +""" + pytype(name, bases, dict) + +Create a new type. Equivalent to `type(name, bases, dict)` in Python. + +If `bases` is not a Python object, it is converted to one using `pytuple`. + +The `dict` may either by a Python object or a Julia iterable. In the latter case, each item +may either be a `name => value` pair or a Python object with a `__name__` attribute. + +In order to use a Julia `Function` as an instance method, it must be wrapped into a Python +function with [`pyfunc`](@ref). Similarly, see also [`pyclassmethod`](@ref), +[`pystaticmethod`](@ref) or [`pyproperty`](@ref). In all these cases, the arguments passed +to the function always have type `Py`. See the example below. + +# Example + +```julia +Foo = pytype("Foo", (), [ + "__module__" => "__main__", + + pyfunc( + name = "__init__", + doc = \"\"\" + Specify x and y to store in the Foo. + + If omitted, y defaults to None. + \"\"\", + function (self, x, y = nothing) + self.x = x + self.y = y + return + end, + ), + + pyfunc( + name = "__repr__", + self -> "Foo(\$(self.x), \$(self.y))", + ), + + pyclassmethod( + name = "frompair", + doc = "Construct a Foo from a tuple of length two.", + (cls, xy) -> cls(xy...), + ), + + pystaticmethod( + name = "hello", + doc = "Prints a friendly greeting.", + (name) -> println("Hello, \$name"), + ), + + "xy" => pyproperty( + doc = "A tuple of x and y.", + get = (self) -> (self.x, self.y), + set = function (self, xy) + (x, y) = xy + self.x = x + self.y = y + nothing + end, + ), +]) +``` +""" +function pytype(name, bases, dict) + bases2 = ispy(bases) ? bases : pytuple(bases) + dict2 = ispy(dict) ? dict : pydict(ispy(item) ? (pygetattr(item, "__name__") => item) : item for item in dict) + pybuiltins.type(name, bases2, dict2) +end + +pyistype(x) = pytypecheckfast(x, C.Py_TPFLAGS_TYPE_SUBCLASS) + +pytypecheck(x, t) = (@autopy x t C.Py_TypeCheck(getptr(x_), getptr(t_))) == 1 +pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(getptr(x_), f)) == 1 + +### slice + +""" + pyslice([start], stop, [step]) + +Construct a Python `slice`. Unspecified arguments default to `None`. +""" +pyslice(x, y, z=pybuiltins.None) = pynew(errcheck(@autopy x y z C.PySlice_New(getptr(x_), getptr(y_), getptr(z_)))) +pyslice(y) = pyslice(pybuiltins.None, y, pybuiltins.None) +export pyslice + +pyisslice(x) = pytypecheck(x, pybuiltins.slice) + +### range + +""" + pyrange([[start], [stop]], [step]) + +Construct a Python `range`. Unspecified arguments default to `None`. +""" +pyrange(x, y, z) = pybuiltins.range(x, y, z) +pyrange(x, y) = pybuiltins.range(x, y) +pyrange(y) = pybuiltins.range(y) +export pyrange + +pyrange_fromrange(x::AbstractRange) = pyrange(first(x), last(x) + sign(step(x)), step(x)) + +pyisrange(x) = pytypecheck(x, pybuiltins.range) + +### tuple + +pynulltuple(len) = pynew(errcheck(C.PyTuple_New(len))) + +function pytuple_setitem(xs::Py, i, x) + errcheck(C.PyTuple_SetItem(getptr(xs), i, incref(getptr(Py(x))))) + return xs +end + +function pytuple_getitem(xs::Py, i) + Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(getptr(xs), i)))) +end + +function pytuple_fromiter(xs) + sz = Base.IteratorSize(typeof(xs)) + if sz isa Base.HasLength || sz isa Base.HasShape + # length known, e.g. Tuple, Pair, Vector + ans = pynulltuple(length(xs)) + for (i, x) in enumerate(xs) + pytuple_setitem(ans, i-1, x) + end + return ans + else + # length unknown + xs_ = pylist_fromiter(xs) + ans = pylist_astuple(xs_) + pydel!(xs_) + return ans + end +end + +@generated function pytuple_fromiter(xs::Tuple) + n = length(xs.parameters) + code = [] + push!(code, :(ans = pynulltuple($n))) + for i in 1:n + push!(code, :(pytuple_setitem(ans, $(i-1), xs[$i]))) + end + push!(code, :(return ans)) + return Expr(:block, code...) +end + +""" + pytuple(x=()) + +Convert `x` to a Python `tuple`. + +If `x` is a Python object, this is equivalent to `tuple(x)` in Python. +Otherwise `x` must be iterable. +""" +pytuple() = pynulltuple(0) +pytuple(x) = ispy(x) ? pybuiltins.tuple(x) : pytuple_fromiter(x) +export pytuple + +pyistuple(x) = pytypecheckfast(x, C.Py_TPFLAGS_TUPLE_SUBCLASS) + +### list + +pynulllist(len) = pynew(errcheck(C.PyList_New(len))) + +function pylist_setitem(xs::Py, i, x) + errcheck(C.PyList_SetItem(getptr(xs), i, incref(getptr(Py(x))))) + return xs +end + +pylist_append(xs::Py, x) = errcheck(@autopy x C.PyList_Append(getptr(xs), getptr(x_))) + +pylist_astuple(x) = pynew(errcheck(@autopy x C.PyList_AsTuple(getptr(x_)))) + +function pylist_fromiter(xs) + sz = Base.IteratorSize(typeof(xs)) + if sz isa Base.HasLength || sz isa Base.HasShape + # length known + ans = pynulllist(length(xs)) + for (i, x) in enumerate(xs) + pylist_setitem(ans, i-1, x) + end + return ans + else + # length unknown + ans = pynulllist(0) + for x in xs + pylist_append(ans, x) + end + return ans + end +end + +""" + pylist(x=()) + +Convert `x` to a Python `list`. + +If `x` is a Python object, this is equivalent to `list(x)` in Python. +Otherwise `x` must be iterable. +""" +pylist() = pynulllist(0) +pylist(x) = ispy(x) ? pybuiltins.list(x) : pylist_fromiter(x) +export pylist + +""" + pycollist(x::AbstractArray) + +Create a nested Python `list`-of-`list`s from the elements of `x`. For matrices, this is a list of columns. +""" +function pycollist(x::AbstractArray{T,N}) where {T,N} + N == 0 && return pynew(Py(x[])) + d = N + ax = axes(x, d) + ans = pynulllist(length(ax)) + for (i, j) in enumerate(ax) + y = pycollist(selectdim(x, d, j)) + pylist_setitem(ans, i-1, y) + pydel!(y) + end + return ans +end +export pycollist + +""" + pyrowlist(x::AbstractArray) + +Create a nested Python `list`-of-`list`s from the elements of `x`. For matrices, this is a list of rows. +""" +function pyrowlist(x::AbstractArray{T,N}) where {T,N} + ndims(x) == 0 && return pynew(Py(x[])) + d = 1 + ax = axes(x, d) + ans = pynulllist(length(ax)) + for (i, j) in enumerate(ax) + y = pyrowlist(selectdim(x, d, j)) + pylist_setitem(ans, i-1, y) + pydel!(y) + end + return ans +end +export pyrowlist + +### set + +pyset_add(set::Py, x) = (errcheck(@autopy x C.PySet_Add(getptr(set), getptr(x_))); set) + +function pyset_update_fromiter(set::Py, xs) + for x in xs + pyset_add(set, x) + end + return set +end +pyset_fromiter(xs) = pyset_update_fromiter(pyset(), xs) +pyfrozenset_fromiter(xs) = pyset_update_fromiter(pyfrozenset(), xs) + +""" + pyset(x=()) + +Convert `x` to a Python `set`. + +If `x` is a Python object, this is equivalent to `set(x)` in Python. +Otherwise `x` must be iterable. +""" +pyset() = pynew(errcheck(C.PySet_New(C.PyNULL))) +pyset(x) = ispy(x) ? pybuiltins.set(x) : pyset_fromiter(x) +export pyset + +""" + pyfrozenset(x=()) + +Convert `x` to a Python `frozenset`. + +If `x` is a Python object, this is equivalent to `frozenset(x)` in Python. +Otherwise `x` must be iterable. +""" +pyfrozenset() = pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) +pyfrozenset(x) = ispy(x) ? pybuiltins.frozenset(x) : pyfrozenset_fromiter(x) +export pyfrozenset + +### dict + +pydict_setitem(x::Py, k, v) = errcheck(@autopy k v C.PyDict_SetItem(getptr(x), getptr(k_), getptr(v_))) + +function pydict_fromiter(kvs) + ans = pydict() + for (k, v) in kvs + pydict_setitem(ans, k, v) + end + return ans +end + +function pystrdict_fromiter(kvs) + ans = pydict() + for (k, v) in kvs + pydict_setitem(ans, string(k), v) + end + return ans +end + +""" + pydict(x) + pydict(; x...) + +Convert `x` to a Python `dict`. In the second form, the keys are strings. + +If `x` is a Python object, this is equivalent to `dict(x)` in Python. +Otherwise `x` must iterate over key-value pairs. +""" +pydict(; kwargs...) = isempty(kwargs) ? pynew(errcheck(C.PyDict_New())) : pystrdict_fromiter(kwargs) +pydict(x) = ispy(x) ? pybuiltins.dict(x) : pydict_fromiter(x) +pydict(x::NamedTuple) = pydict(; x...) +export pydict + +### datetime + +# We used to use 1/1/1 but pandas.Timestamp is a subclass of datetime and does not include +# this date, so we use 1970 instead. +const _base_datetime = DateTime(1970, 1, 1) +const _base_pydatetime = pynew() + +function init_datetime() + pycopy!(_base_pydatetime, pydatetimetype(1970, 1, 1)) +end + +pydate(year, month, day) = pydatetype(year, month, day) +pydate(x::Date) = pydate(year(x), month(x), day(x)) +export pydate + +pytime(_hour=0, _minute=0, _second=0, _microsecond=0, _tzinfo=nothing; hour=_hour, minute=_minute, second=_second, microsecond=_microsecond, tzinfo=_tzinfo, fold=0) = pytimetype(hour, minute, second, microsecond, tzinfo, fold=fold) +pytime(x::Time) = + if iszero(nanosecond(x)) + pytime(hour(x), minute(x), second(x), millisecond(x) * 1000 + microsecond(x)) + else + errset(pybuiltins.ValueError, "cannot create 'datetime.time' with less than microsecond resolution") + pythrow() + end +export pytime + +pydatetime(year, month, day, _hour=0, _minute=0, _second=0, _microsecond=0, _tzinfo=nothing; hour=_hour, minute=_minute, second=_second, microsecond=_microsecond, tzinfo=_tzinfo, fold=0) = pydatetimetype(year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold) +function pydatetime(x::DateTime) + # compute time since _base_datetime + # this accounts for fold + d = pytimedeltatype(milliseconds = (x - _base_datetime).value) + ans = _base_pydatetime + d + pydel!(d) + return ans +end +pydatetime(x::Date) = pydatetime(year(x), month(x), day(x)) +export pydatetime + +function pytime_isaware(x) + tzinfo = pygetattr(x, "tzinfo") + if pyisnone(tzinfo) + pydel!(tzinfo) + return false + end + utcoffset = tzinfo.utcoffset + pydel!(tzinfo) + o = utcoffset(nothing) + pydel!(utcoffset) + ans = !pyisnone(o) + pydel!(o) + return ans +end + +function pydatetime_isaware(x) + tzinfo = pygetattr(x, "tzinfo") + if pyisnone(tzinfo) + pydel!(tzinfo) + return false + end + utcoffset = tzinfo.utcoffset + pydel!(tzinfo) + o = utcoffset(x) + pydel!(utcoffset) + ans = !pyisnone(o) + pydel!(o) + return ans +end + +### fraction + +pyfraction(x::Rational) = pyfraction(numerator(x), denominator(x)) +pyfraction(x, y) = pyfractiontype(x, y) +pyfraction(x) = pyfractiontype(x) +pyfraction() = pyfractiontype() +export pyfraction + +### eval/exec + +const MODULE_GLOBALS = Dict{Module,Py}() + +function _pyeval_args(code, globals, locals) + if code isa AbstractString + code_ = code + elseif ispy(code) + code_ = code + else + throw(ArgumentError("code must be a string or Python code")) + end + if globals isa Module + globals_ = get!(pydict, MODULE_GLOBALS, globals) + elseif ispy(globals) + globals_ = globals + else + throw(ArgumentError("globals must be a module or a Python dict")) + end + if locals === nothing + locals_ = pynew(Py(globals_)) + elseif ispy(locals) + locals_ = pynew(Py(locals)) + else + locals_ = pydict(locals) + end + return (code_, globals_, locals_) +end + +""" + pyeval([T=Py], code, globals, locals=nothing) + +Evaluate the given Python `code`, returning the result as a `T`. + +If `globals` is a `Module`, then a persistent `dict` unique to that module is used. + +By default the code runs in global scope (i.e. `locals===globals`). To use a temporary +local scope, set `locals` to `()`, or to a `NamedTuple` of variables to include in the +scope. + +See also [`@pyeval`](@ref). + +# Examples + +The following computes `1.1+2.2` in the `Main` module as a `Float64`: +``` +pyeval(Float64, "x+y", Main, (x=1.1, y=2.2)) # returns 3.3 +``` +""" +function pyeval(::Type{T}, code, globals, locals=nothing) where {T} + code_, globals_, locals_ = _pyeval_args(code, globals, locals) + ans = pybuiltins.eval(code_, globals_, locals_) + pydel!(locals_) + return pyconvert(T, ans) +end +pyeval(code, globals, locals=nothing) = pyeval(Py, code, globals, locals) +export pyeval + +_pyexec_ans(::Type{Nothing}, globals, locals) = nothing +@generated function _pyexec_ans(::Type{NamedTuple{names, types}}, globals, locals) where {names, types} + # TODO: use precomputed interned strings + # TODO: try to load from globals too + n = length(names) + code = [] + vars = Symbol[] + for i in 1:n + v = Symbol(:ans, i) + push!(vars, v) + push!(code, :($v = pyconvert($(types.parameters[i]), pygetitem(locals, $(string(names[i])))))) + end + push!(code, :(return $(NamedTuple{names, types})(($(vars...),)))) + return Expr(:block, code...) +end + +""" + pyexec([T=Nothing], code, globals, locals=nothing) + +Execute the given Python `code`. + +If `globals` is a `Module`, then a persistent `dict` unique to that module is used. + +By default the code runs in global scope (i.e. `locals===globals`). To use a temporary +local scope, set `locals` to `()`, or to a `NamedTuple` of variables to include in the +scope. + +If `T==Nothing` then returns `nothing`. Otherwise `T` must be a concrete `NamedTuple` type +and the corresponding items from `locals` are extracted and returned. + +See also [`@pyexec`](@ref). + +# Examples + +The following computes `1.1+2.2` in the `Main` module as a `Float64`: +``` +pyexec(@NamedTuple{ans::Float64}, "ans=x+y", Main, (x=1.1, y=2.2)) # returns (ans = 3.3,) +``` + +Marking variables as `global` saves them into the module scope, so that they are available +in subsequent invocations: +``` +pyexec("global x; x=12", Main) +pyeval(Int, "x", Main) # returns 12 +``` +""" +function pyexec(::Type{T}, code, globals, locals=nothing) where {T} + code_, globals_, locals_ = _pyeval_args(code, globals, locals) + pydel!(pybuiltins.exec(code_, globals_, locals_)) + ans = _pyexec_ans(T, globals_, locals_) + pydel!(locals_) + return ans +end +pyexec(code, globals, locals=nothing) = pyexec(Nothing, code, globals, locals) +export pyexec + +function _pyeval_macro_code(arg) + if arg isa String + return arg + elseif arg isa Expr && arg.head === :macrocall && arg.args[1] == :(`foo`).args[1] + return arg.args[3] + else + return nothing + end +end + +function _pyeval_macro_args(arg, filename, mode) + # separate out inputs => code => outputs (with only code being required) + if @capture(arg, inputs_ => code_ => outputs_) + code = _pyeval_macro_code(code) + code === nothing && error("invalid code") + elseif @capture(arg, lhs_ => rhs_) + code = _pyeval_macro_code(lhs) + if code === nothing + code = _pyeval_macro_code(rhs) + code === nothing && error("invalid code") + inputs = lhs + outputs = nothing + else + inputs = nothing + outputs = rhs + end + else + code = _pyeval_macro_code(arg) + code === nothing && error("invalid code") + inputs = outputs = nothing + end + # precompile the code + codestr = code + codeobj = pynew() + codeready = Ref(false) + code = quote + if !$codeready[] + $pycopy!($codeobj, $pybuiltins.compile($codestr, $filename, $mode)) + $codeready[] = true + end + $codeobj + end + # convert inputs to locals + if inputs === nothing + locals = () + else + if inputs isa Expr && inputs.head === :tuple + inputs = inputs.args + else + inputs = [inputs] + end + locals = [] + for input in inputs + if @capture(input, var_Symbol) + push!(locals, var => var) + elseif @capture(input, var_Symbol = ex_) + push!(locals, var => ex) + else + error("invalid input: $input") + end + end + locals = :(($([:($var = $ex) for (var,ex) in locals]...),)) + end + # done + return locals, code, outputs +end + +""" + @pyeval [inputs =>] code [=> T] + +Evaluate the given `code` in a new local scope and return the answer as a `T`. + +The global scope is persistent and unique to the current module. + +The `code` must be a literal string or command. + +The `inputs` is a tuple of inputs of the form `v=expr` to be included in the local scope. +Only `v` is required, `expr` defaults to `v`. + +# Examples + +The following computes `1.1+2.2` and returns a `Float64`: +``` +@pyeval (x=1.1, y=2.2) => `x+y` => Float64 # returns 3.3 +``` +""" +macro pyeval(arg) + locals, code, outputs = _pyeval_macro_args(arg, "$(__source__.file):$(__source__.line)", "eval") + if outputs === nothing + outputs = Py + end + esc(:($pyeval($outputs, $code, $__module__, $locals))) +end +export @pyeval + +""" + @pyexec [inputs =>] code [=> outputs] + +Execute the given `code` in a new local scope. + +The global scope is persistent and unique to the current module. + +The `code` must be a literal string or command. + +The `inputs` is a tuple of inputs of the form `v=expr` to be included in the local scope. +Only `v` is required, `expr` defaults to `v`. + +The `outputs` is a tuple of outputs of the form `x::T=v`, meaning that `v` is extracted from +locals, converted to `T` and assigned to `x`. Only `x` is required: `T` defaults to `Py` +and `v` defaults to `x`. + +# Examples + +The following computes `1.1+2.2` and assigns its value to `ans` as a `Float64`: +``` +@pyexec (x=1.1, y=2.2) => `ans=x+y` => ans::Float64 # returns 3.3 +``` + +Marking variables as `global` saves them into the module scope, so that they are available +in subsequent invocations: +``` +@pyexec `global x; x=12` +@pyeval `x` => Int # returns 12 +``` +""" +macro pyexec(arg) + locals, code, outputs = _pyeval_macro_args(arg, "$(__source__.file):$(__source__.line)", "exec") + if outputs === nothing + outputs = Nothing + esc(:($pyexec(Nothing, $code, $__module__, $locals))) + else + if outputs isa Expr && outputs.head === :tuple + oneoutput = false + outputs = outputs.args + else + oneoutput = true + outputs = [outputs] + end + pyvars = Symbol[] + jlvars = Symbol[] + types = [] + for output in outputs + if @capture(output, lhs_ = rhs_) + rhs isa Symbol || error("invalid output: $output") + output = lhs + pyvar = rhs + else + pyvar = missing + end + if @capture(output, lhs_ :: rhs_) + outtype = rhs + output = lhs + else + outtype = Py + end + output isa Symbol || error("invalid output: $output") + if pyvar === missing + pyvar = output + end + push!(pyvars, pyvar) + push!(jlvars, output) + push!(types, outtype) + end + outtype = :($NamedTuple{($(map(QuoteNode, pyvars)...),), Tuple{$(types...),}}) + ans = :($pyexec($outtype, $code, $__module__, $locals)) + if oneoutput + ans = :($(jlvars[1]) = $ans[1]) + else + if pyvars != jlvars + outtype2 = :($NamedTuple{($(map(QuoteNode, jlvars)...),), Tuple{$(types...),}}) + ans = :($outtype2($ans)) + end + ans = :(($(jlvars...),) = $ans) + end + esc(ans) + end +end +export @pyexec + +### with + +""" + pywith(f, o, d=nothing) + +Equivalent to `with o as x: f(x)` in Python, where `x` is a `Py`. + +On success, the value of `f(x)` is returned. + +If an exception occurs but is suppressed then `d` is returned. +""" +function pywith(f, o, d = nothing) + o = Py(o) + t = pytype(o) + exit = t.__exit__ + value = t.__enter__(o) + exited = false + try + return f(value) + catch exc + if exc isa PyException + exited = true + if pytruth(exit(o, exc.t, exc.v, exc.b)) + return d + end + end + rethrow() + finally + exited || exit(o, pybuiltins.None, pybuiltins.None, pybuiltins.None) + end +end +export pywith + +### import + +""" + pyimport(m) + pyimport(m => k) + pyimport(m => (k1, k2, ...)) + pyimport(m1, m2, ...) + +Import a module `m`, or an attribute `k`, or a tuple of attributes. + +If several arguments are given, return the results of importing each one in a tuple. +""" +pyimport(m) = pynew(errcheck(@autopy m C.PyImport_Import(getptr(m_)))) +pyimport((m,k)::Pair) = (m_=pyimport(m); k_=pygetattr(m_,k); pydel!(m_); k_) +pyimport((m,ks)::Pair{<:Any,<:Tuple}) = (m_=pyimport(m); ks_=map(k->pygetattr(m_,k), ks); pydel!(m_); ks_) +pyimport(m1, m2, ms...) = map(pyimport, (m1, m2, ms...)) +export pyimport + +### builtins not covered elsewhere + +""" + pyprint(...) + +Equivalent to `print(...)` in Python. +""" +pyprint(args...; kwargs...) = (pydel!(pybuiltins.print(args...; kwargs...)); nothing) +export pyprint + +function _pyhelp(args...) + pyisnone(pybuiltins.help) && error("Python help is not available") + pydel!(pybuiltins.help(args...)) + nothing +end +""" + pyhelp([x]) + +Equivalent to `help(x)` in Python. +""" +pyhelp() = _pyhelp() +pyhelp(x) = _pyhelp(x) +export pyhelp + +""" + pyall(x) + +Equivalent to `all(x)` in Python. +""" +function pyall(x) + y = pybuiltins.all(x) + z = pybool_asbool(y) + pydel!(y) + z +end +export pyall + +""" + pyany(x) + +Equivalent to `any(x)` in Python. +""" +function pyany(x) + y = pybuiltins.any(x) + z = pybool_asbool(y) + pydel!(y) + z +end +export pyany + +""" + pycallable(x) + +Equivalent to `callable(x)` in Python. +""" +function pycallable(x) + y = pybuiltins.callable(x) + z = pybool_asbool(y) + pydel!(y) + z +end +export pycallable + +""" + pycompile(...) + +Equivalent to `compile(...)` in Python. +""" +pycompile(args...; kwargs...) = pybuiltins.compile(args...; kwargs...) +export pycompile diff --git a/src/config.jl b/src/Py/config.jl similarity index 65% rename from src/config.jl rename to src/Py/config.jl index 0c0077ba..53eb2454 100644 --- a/src/config.jl +++ b/src/Py/config.jl @@ -1,9 +1,7 @@ -Base.@kwdef mutable struct Config +@kwdef mutable struct Config meta :: String = "" auto_sys_last_traceback :: Bool = true auto_fix_qt_plugin_path :: Bool = true end const CONFIG = Config() - -# TODO: load_config(), save_config() diff --git a/src/concrete/consts.jl b/src/Py/consts.jl similarity index 100% rename from src/concrete/consts.jl rename to src/Py/consts.jl diff --git a/src/err.jl b/src/Py/err.jl similarity index 98% rename from src/err.jl rename to src/Py/err.jl index 7c667acf..37f67748 100644 --- a/src/err.jl +++ b/src/Py/err.jl @@ -70,8 +70,6 @@ export PyException ispy(x::PyException) = true Py(x::PyException) = x.v -pyconvert_rule_exception(::Type{R}, x::Py) where {R<:PyException} = pyconvert_return(PyException(x)) - function Base.show(io::IO, x::PyException) show(io, typeof(x)) print(io, "(") diff --git a/src/gc.jl b/src/Py/gc.jl similarity index 98% rename from src/gc.jl rename to src/Py/gc.jl index a8042b82..76c7ed21 100644 --- a/src/gc.jl +++ b/src/Py/gc.jl @@ -5,7 +5,7 @@ See `disable` and `enable`. """ module GC -import ..PythonCall.C +using .._Py: C const ENABLED = Ref(true) const QUEUE = C.PyPtr[] diff --git a/src/juliacall.jl b/src/Py/juliacall.jl similarity index 86% rename from src/juliacall.jl rename to src/Py/juliacall.jl index e0675818..15945f6b 100644 --- a/src/juliacall.jl +++ b/src/Py/juliacall.jl @@ -1,5 +1,6 @@ const pyjuliacallmodule = pynew() const pyJuliaError = pynew() +const CPyExc_JuliaError = Ref(C.PyNULL) function init_juliacall() # ensure the 'juliacall' module exists @@ -28,15 +29,6 @@ function init_juliacall() @assert pystr_asstring(jl.__version__) == string(VERSION) @assert !pybool_asbool(jl.CONFIG["init"]) end -end - -function init_juliacall_2() - jl = pyjuliacallmodule - jl.Main = Main - jl.Core = Core - jl.Base = Base - jl.Pkg = Pkg - jl.PythonCall = PythonCall pycopy!(pyJuliaError, jl.JuliaError) - C.POINTERS.PyExc_JuliaError = incref(getptr(pyJuliaError)) + CPyExc_JuliaError[] = incref(getptr(pyJuliaError)) end diff --git a/src/pyconst_macro.jl b/src/Py/pyconst_macro.jl similarity index 100% rename from src/pyconst_macro.jl rename to src/Py/pyconst_macro.jl diff --git a/src/compat/stdlib.jl b/src/Py/stdlib.jl similarity index 95% rename from src/compat/stdlib.jl rename to src/Py/stdlib.jl index a4c6a868..89024a6c 100644 --- a/src/compat/stdlib.jl +++ b/src/Py/stdlib.jl @@ -3,7 +3,7 @@ const pymodulehooks = pynew() function init_stdlib() # check word size - pywordsize = @py(jlbool(pysysmodule.maxsize > 2^32)) ? 64 : 32 + pywordsize = pygt(Bool, pysysmodule.maxsize, Int64(2)^32) ? 64 : 32 pywordsize == Sys.WORD_SIZE || error("Julia is $(Sys.WORD_SIZE)-bit but Python is $(pywordsize)-bit") if C.CTX.is_embedded @@ -43,8 +43,8 @@ function init_stdlib() end # add hook to perform certain actions when certain modules are loaded - @py g = {} - @py @exec """ + g = pydict() + pyexec(""" import sys class JuliaCompatHooks: def __init__(self): @@ -63,7 +63,7 @@ function init_stdlib() h() JULIA_COMPAT_HOOKS = JuliaCompatHooks() sys.meta_path.insert(0, JULIA_COMPAT_HOOKS) - """ g + """, g) pycopy!(pymodulehooks, g["JULIA_COMPAT_HOOKS"]) end diff --git a/src/PythonCall.jl b/src/PythonCall.jl index b76b215c..d536ef14 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -3,138 +3,36 @@ module PythonCall const VERSION = v"0.9.15" const ROOT_DIR = dirname(@__DIR__) -using Base: @propagate_inbounds -using MacroTools, Dates, Tables, Markdown, Serialization, Requires, Pkg, REPL - -include("utils.jl") - -include("cpython/CPython.jl") - -include("gc.jl") -include("Py.jl") -include("err.jl") -include("config.jl") -include("convert.jl") -# abstract interfaces -include("abstract/object.jl") -include("abstract/iter.jl") -include("abstract/builtins.jl") -include("abstract/number.jl") -include("abstract/collection.jl") -# concrete types -include("concrete/import.jl") -include("concrete/consts.jl") -include("concrete/str.jl") -include("concrete/bytes.jl") -include("concrete/tuple.jl") -include("concrete/list.jl") -include("concrete/dict.jl") -include("concrete/bool.jl") -include("concrete/int.jl") -include("concrete/float.jl") -include("concrete/complex.jl") -include("concrete/set.jl") -include("concrete/slice.jl") -include("concrete/range.jl") -include("concrete/none.jl") -include("concrete/type.jl") -include("concrete/fraction.jl") -include("concrete/datetime.jl") -include("concrete/code.jl") -include("concrete/ctypes.jl") -include("concrete/numpy.jl") -include("concrete/pandas.jl") -# @py -# anything below can depend on @py, anything above cannot -include("py_macro.jl") -# jlwrap -include("jlwrap/base.jl") -include("jlwrap/raw.jl") -include("jlwrap/callback.jl") -include("jlwrap/any.jl") -include("jlwrap/module.jl") -include("jlwrap/type.jl") -include("jlwrap/iter.jl") -include("jlwrap/objectarray.jl") -include("jlwrap/array.jl") -include("jlwrap/vector.jl") -include("jlwrap/dict.jl") -include("jlwrap/set.jl") -include("jlwrap/number.jl") -include("jlwrap/io.jl") -# pywrap -include("pywrap/PyIterable.jl") -include("pywrap/PyList.jl") -include("pywrap/PySet.jl") -include("pywrap/PyDict.jl") -include("pywrap/PyArray.jl") -include("pywrap/PyIO.jl") -include("pywrap/PyTable.jl") -include("pywrap/PyPandasDataFrame.jl") -# misc -include("pyconst_macro.jl") -include("juliacall.jl") -include("compat/stdlib.jl") -include("compat/with.jl") -include("compat/multimedia.jl") -include("compat/serialization.jl") -include("compat/gui.jl") -include("compat/ipython.jl") -include("compat/tables.jl") - -function __init__() - C.with_gil() do - init_consts() - init_pyconvert() - init_datetime() - # juliacall/jlwrap - init_juliacall() - init_jlwrap_base() - init_jlwrap_raw() - init_jlwrap_callback() - init_jlwrap_any() - init_jlwrap_module() - init_jlwrap_type() - init_jlwrap_iter() - init_jlwrap_array() - init_jlwrap_vector() - init_jlwrap_dict() - init_jlwrap_set() - init_jlwrap_number() - init_jlwrap_io() - init_juliacall_2() - # compat - init_stdlib() - init_pyshow() - init_gui() - init_tables() - init_ctypes() - init_numpy() - init_pandas() +include("utils/_.jl") +include("CPython/_.jl") +include("Py/_.jl") +include("pyconvert/_.jl") +include("pymacro/_.jl") +include("pywrap/_.jl") +include("jlwrap/_.jl") +include("compat/_.jl") + +# re-export everything +for m in [:_Py, :_pyconvert, :_pymacro, :_pywrap, :_jlwrap, :_compat] + for k in names(@eval($m)) + if k != m + @eval using .$m: $k + @eval export $k + end end - @require PyCall="438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall) end -function init_pycall(PyCall::Module) - # allow explicit conversion between PythonCall.Py and PyCall.PyObject - # provided they are using the same interpretr - errmsg = """ - Conversion between `PyCall.PyObject` and `PythonCall.Py` is only possible when using the same Python interpreter. +# non-exported API +for k in [:C, :GC, :pynew, :pyisnull, :pycopy!, :getptr, :pydel!, :unsafe_pynext, :PyNULL] + @eval const $k = _Py.$k +end +for k in [:pyconvert_add_rule, :pyconvert_return, :pyconvert_unconverted, :PYCONVERT_PRIORITY_WRAP, :PYCONVERT_PRIORITY_ARRAY, :PYCONVERT_PRIORITY_CANONICAL, :PYCONVERT_PRIORITY_NORMAL, :PYCONVERT_PRIORITY_FALLBACK] + @eval const $k = _pyconvert.$k +end - There are two ways to achieve this: - - Set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`. This forces PythonCall to use the same - interpreter as PyCall, but PythonCall loses the ability to manage its own dependencies. - - Set the environment variable `PYTHON` to `PythonCall.C.CTX.exe_path` and rebuild PyCall. This forces PyCall - to use the same interpreter as PythonCall, but needs to be repeated whenever you switch Julia environment. - """ - @eval function Py(x::$PyCall.PyObject) - C.CTX.matches_pycall::Bool || error($errmsg) - return pynew(C.PyPtr($PyCall.pyreturn(x))) - end - @eval function $PyCall.PyObject(x::Py) - C.CTX.matches_pycall::Bool || error($errmsg) - return $PyCall.PyObject($PyCall.PyPtr(incref(getptr(x)))) - end +# not API but used in tests +for k in [:pyjlanytype, :pyjlarraytype, :pyjlvectortype, :pyjlbinaryiotype, :pyjltextiotype, :pyjldicttype, :pyjlmoduletype, :pyjlintegertype, :pyjlrationaltype, :pyjlrealtype, :pyjlcomplextype, :pyjlsettype, :pyjltypetype] + @eval const $k = _jlwrap.$k end end diff --git a/src/abstract/builtins.jl b/src/abstract/builtins.jl deleted file mode 100644 index 23a648d3..00000000 --- a/src/abstract/builtins.jl +++ /dev/null @@ -1,70 +0,0 @@ -# Builtin functions not covered elsewhere - -""" - pyprint(...) - -Equivalent to `print(...)` in Python. -""" -pyprint(args...; kwargs...) = (pydel!(pybuiltins.print(args...; kwargs...)); nothing) -export pyprint - -function _pyhelp(args...) - pyisnone(pybuiltins.help) && error("Python help is not available") - pydel!(pybuiltins.help(args...)) - nothing -end -""" - pyhelp([x]) - -Equivalent to `help(x)` in Python. -""" -pyhelp() = _pyhelp() -pyhelp(x) = _pyhelp(x) -export pyhelp - -""" - pyall(x) - -Equivalent to `all(x)` in Python. -""" -function pyall(x) - y = pybuiltins.all(x) - z = pybool_asbool(y) - pydel!(y) - z -end -export pyall - -""" - pyany(x) - -Equivalent to `any(x)` in Python. -""" -function pyany(x) - y = pybuiltins.any(x) - z = pybool_asbool(y) - pydel!(y) - z -end -export pyany - -""" - pycallable(x) - -Equivalent to `callable(x)` in Python. -""" -function pycallable(x) - y = pybuiltins.callable(x) - z = pybool_asbool(y) - pydel!(y) - z -end -export pycallable - -""" - pycompile(...) - -Equivalent to `compile(...)` in Python. -""" -pycompile(args...; kwargs...) = pybuiltins.compile(args...; kwargs...) -export pycompile diff --git a/src/abstract/collection.jl b/src/abstract/collection.jl deleted file mode 100644 index 0c093db0..00000000 --- a/src/abstract/collection.jl +++ /dev/null @@ -1,197 +0,0 @@ -# Vector - -function _pyconvert_rule_iterable(ans::Vector{T0}, it::Py, ::Type{T1}) where {T0,T1} - @label again - x_ = unsafe_pynext(it) - if pyisnull(x_) - pydel!(it) - return pyconvert_return(ans) - end - x = @pyconvert(T1, x_) - if x isa T0 - push!(ans, x) - @goto again - end - T2 = Utils._promote_type_bounded(T0, typeof(x), T1) - ans2 = Vector{T2}(ans) - push!(ans2, x) - return _pyconvert_rule_iterable(ans2, it, T1) -end - -function pyconvert_rule_iterable(::Type{R}, x::Py, ::Type{Vector{T0}}=Utils._type_lb(R), ::Type{Vector{T1}}=Utils._type_ub(R)) where {R<:Vector,T0,T1} - it = pyiter(x) - ans = Vector{T0}() - return _pyconvert_rule_iterable(ans, it, T1) -end - -# Set - -function _pyconvert_rule_iterable(ans::Set{T0}, it::Py, ::Type{T1}) where {T0,T1} - @label again - x_ = unsafe_pynext(it) - if pyisnull(x_) - pydel!(it) - return pyconvert_return(ans) - end - x = @pyconvert(T1, x_) - if x isa T0 - push!(ans, x) - @goto again - end - T2 = Utils._promote_type_bounded(T0, typeof(x), T1) - ans2 = Set{T2}(ans) - push!(ans2, x) - return _pyconvert_rule_iterable(ans2, it, T1) -end - -function pyconvert_rule_iterable(::Type{R}, x::Py, ::Type{Set{T0}}=Utils._type_lb(R), ::Type{Set{T1}}=Utils._type_ub(R)) where {R<:Set,T0,T1} - it = pyiter(x) - ans = Set{T0}() - return _pyconvert_rule_iterable(ans, it, T1) -end - -# Dict - -function _pyconvert_rule_mapping(ans::Dict{K0,V0}, x::Py, it::Py, ::Type{K1}, ::Type{V1}) where {K0,V0,K1,V1} - @label again - k_ = unsafe_pynext(it) - if pyisnull(k_) - pydel!(it) - return pyconvert_return(ans) - end - v_ = pygetitem(x, k_) - k = @pyconvert(K1, k_) - v = @pyconvert(V1, v_) - if k isa K0 && v isa V0 - push!(ans, k => v) - @goto again - end - K2 = Utils._promote_type_bounded(K0, typeof(k), K1) - V2 = Utils._promote_type_bounded(V0, typeof(v), V1) - ans2 = Dict{K2,V2}(ans) - push!(ans2, k => v) - return _pyconvert_rule_mapping(ans2, x, it, K1, V1) -end - -function pyconvert_rule_mapping(::Type{R}, x::Py, ::Type{Dict{K0,V0}}=Utils._type_lb(R), ::Type{Dict{K1,V1}}=Utils._type_ub(R)) where {R<:Dict,K0,V0,K1,V1} - it = pyiter(x) - ans = Dict{K0,V0}() - return _pyconvert_rule_mapping(ans, x, it, K1, V1) -end - -# Tuple - -function pyconvert_rule_iterable(::Type{T}, xs::Py) where {T<:Tuple} - T isa DataType || return pyconvert_unconverted() - if T != Tuple{} && Tuple{T.parameters[end]} == Base.tuple_type_tail(Tuple{T.parameters[end]}) - isvararg = true - vartype = Base.tuple_type_head(Tuple{T.parameters[end]}) - ts = T.parameters[1:end-1] - else - isvararg = false - vartype = Union{} - ts = T.parameters - end - zs = Any[] - for x in xs - if length(zs) < length(ts) - t = ts[length(zs) + 1] - elseif isvararg - t = vartype - else - return pyconvert_unconverted() - end - z = @pyconvert(t, x) - push!(zs, z) - end - return length(zs) < length(ts) ? pyconvert_unconverted() : pyconvert_return(T(zs)) -end - -for N in 0:16 - Ts = [Symbol("T", n) for n in 1:N] - zs = [Symbol("z", n) for n in 1:N] - # Tuple with N elements - @eval function pyconvert_rule_iterable(::Type{Tuple{$(Ts...)}}, xs::Py) where {$(Ts...)} - xs = pytuple(xs) - n = pylen(xs) - n == $N || return pyconvert_unconverted() - $(( - :($z = @pyconvert($T, pytuple_getitem(xs, $(i-1)))) - for (i, T, z) in zip(1:N, Ts, zs) - )...) - return pyconvert_return(($(zs...),)) - end - # Tuple with N elements plus Vararg - @eval function pyconvert_rule_iterable(::Type{Tuple{$(Ts...),Vararg{V}}}, xs::Py) where {$(Ts...),V} - xs = pytuple(xs) - n = pylen(xs) - n ≥ $N || return pyconvert_unconverted() - $(( - :($z = @pyconvert($T, pytuple_getitem(xs, $(i-1)))) - for (i, T, z) in zip(1:N, Ts, zs) - )...) - vs = V[] - for i in $(N+1):n - v = @pyconvert(V, pytuple_getitem(xs, i-1)) - push!(vs, v) - end - return pyconvert_return(($(zs...), vs...)) - end -end - -# Pair - -function pyconvert_rule_iterable(::Type{R}, x::Py, ::Type{Pair{K0,V0}}=Utils._type_lb(R), ::Type{Pair{K1,V1}}=Utils._type_ub(R)) where {R<:Pair,K0,V0,K1,V1} - it = pyiter(x) - k_ = unsafe_pynext(it) - if pyisnull(k_) - pydel!(it) - pydel!(k_) - return pyconvert_unconverted() - end - k = @pyconvert(K1, k_) - v_ = unsafe_pynext(it) - if pyisnull(v_) - pydel!(it) - pydel!(v_) - return pyconvert_unconverted() - end - v = @pyconvert(V1, v_) - z_ = unsafe_pynext(it) - pydel!(it) - if pyisnull(z_) - pydel!(z_) - else - pydel!(z_) - return pyconvert_unconverted() - end - K2 = Utils._promote_type_bounded(K0, typeof(k), K1) - V2 = Utils._promote_type_bounded(V0, typeof(v), V1) - return pyconvert_return(Pair{K2,V2}(k, v)) -end - -# NamedTuple - -_nt_names_types(::Type) = nothing -_nt_names_types(::Type{NamedTuple}) = (nothing, nothing) -_nt_names_types(::Type{NamedTuple{names}}) where {names} = (names, nothing) -_nt_names_types(::Type{NamedTuple{names,types} where {names}}) where {types} = (nothing, types) -_nt_names_types(::Type{NamedTuple{names,types}}) where {names,types} = (names, types) - -function pyconvert_rule_iterable(::Type{R}, x::Py) where {R<:NamedTuple} - # this is actually strict and only converts python named tuples (i.e. tuples with a - # _fields attribute) where the field names match those from R (if specified). - names_types = _nt_names_types(R) - names_types === nothing && return pyconvert_unconverted() - names, types = names_types - PythonCall.pyistuple(x) || return pyconvert_unconverted() - names2_ = pygetattr(x, "_fields", pybuiltins.None) - names2 = @pyconvert(names === nothing ? Tuple{Vararg{Symbol}} : typeof(names), names2_) - pydel!(names2_) - names === nothing || names === names2 || return pyconvert_unconverted() - types2 = types === nothing ? NTuple{length(names2),Any} : types - vals = @pyconvert(types2, x) - length(vals) == length(names2) || return pyconvert_unconverted() - types3 = types === nothing ? typeof(vals) : types - return pyconvert_return(NamedTuple{names2,types3}(vals)) -end diff --git a/src/abstract/iter.jl b/src/abstract/iter.jl deleted file mode 100644 index 129e0f7e..00000000 --- a/src/abstract/iter.jl +++ /dev/null @@ -1,22 +0,0 @@ -""" - pyiter(x) - -Equivalent to `iter(x)` in Python. -""" -pyiter(x) = pynew(errcheck(@autopy x C.PyObject_GetIter(getptr(x_)))) -export pyiter - -""" - pynext(x) - -Equivalent to `next(x)` in Python. -""" -pynext(x) = pybuiltins.next(x) -export pynext - -""" - unsafe_pynext(x) - -Return the next item in the iterator `x`. When there are no more items, return NULL. -""" -unsafe_pynext(x::Py) = Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(getptr(x)))) diff --git a/src/abstract/number.jl b/src/abstract/number.jl deleted file mode 100644 index 6b347216..00000000 --- a/src/abstract/number.jl +++ /dev/null @@ -1,203 +0,0 @@ -# unary -""" - pyneg(x) - -Equivalent to `-x` in Python. -""" -pyneg(x) = pynew(errcheck(@autopy x C.PyNumber_Negative(getptr(x_)))) -""" - pypos(x) - -Equivalent to `+x` in Python. -""" -pypos(x) = pynew(errcheck(@autopy x C.PyNumber_Positive(getptr(x_)))) -""" - pyabs(x) - -Equivalent to `abs(x)` in Python. -""" -pyabs(x) = pynew(errcheck(@autopy x C.PyNumber_Absolute(getptr(x_)))) -""" - pyinv(x) - -Equivalent to `~x` in Python. -""" -pyinv(x) = pynew(errcheck(@autopy x C.PyNumber_Invert(getptr(x_)))) -""" - pyindex(x) - -Convert `x` losslessly to an `int`. -""" -pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(getptr(x_)))) -export pyneg, pypos, pyabs, pyinv, pyindex - -# binary -""" - pyadd(x, y) - -Equivalent to `x + y` in Python. -""" -pyadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Add(getptr(x_), getptr(y_)))) -""" - pysub(x, y) - -Equivalent to `x - y` in Python. -""" -pysub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Subtract(getptr(x_), getptr(y_)))) -""" - pymul(x, y) - -Equivalent to `x * y` in Python. -""" -pymul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Multiply(getptr(x_), getptr(y_)))) -""" - pymatmul(x, y) - -Equivalent to `x @ y` in Python. -""" -pymatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(getptr(x_), getptr(y_)))) -""" - pyfloordiv(x, y) - -Equivalent to `x // y` in Python. -""" -pyfloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(getptr(x_), getptr(y_)))) -""" - pytruediv(x, y) - -Equivalent to `x / y` in Python. -""" -pytruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(getptr(x_), getptr(y_)))) -""" - pymod(x, y) - -Equivalent to `x % y` in Python. -""" -pymod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Remainder(getptr(x_), getptr(y_)))) -""" - pydivmod(x, y) - -Equivalent to `divmod(x, y)` in Python. -""" -pydivmod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Divmod(getptr(x_), getptr(y_)))) -""" - pylshift(x, y) - -Equivalent to `x << y` in Python. -""" -pylshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Lshift(getptr(x_), getptr(y_)))) -""" - pyrshift(x, y) - -Equivalent to `x >> y` in Python. -""" -pyrshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Rshift(getptr(x_), getptr(y_)))) -""" - pyand(x, y) - -Equivalent to `x & y` in Python. -""" -pyand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_And(getptr(x_), getptr(y_)))) -""" - pyxor(x, y) - -Equivalent to `x ^ y` in Python. -""" -pyxor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Xor(getptr(x_), getptr(y_)))) -""" - pyor(x, y) - -Equivalent to `x | y` in Python. -""" -pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(getptr(x_), getptr(y_)))) -export pyadd, pysub, pymul, pymatmul, pyfloordiv, pytruediv, pymod, pydivmod, pylshift, pyrshift, pyand, pyxor, pyor - -# binary in-place -""" - pyiadd(x, y) - -In-place add. `x = pyiadd(x, y)` is equivalent to `x += y` in Python. -""" -pyiadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(getptr(x_), getptr(y_)))) -""" - pyisub(x, y) - -In-place subtract. `x = pyisub(x, y)` is equivalent to `x -= y` in Python. -""" -pyisub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(getptr(x_), getptr(y_)))) -""" - pyimul(x, y) - -In-place multiply. `x = pyimul(x, y)` is equivalent to `x *= y` in Python. -""" -pyimul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(getptr(x_), getptr(y_)))) -""" - pyimatmul(x, y) - -In-place matrix multiply. `x = pyimatmul(x, y)` is equivalent to `x @= y` in Python. -""" -pyimatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(getptr(x_), getptr(y_)))) -""" - pyifloordiv(x, y) - -In-place floor divide. `x = pyifloordiv(x, y)` is equivalent to `x //= y` in Python. -""" -pyifloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(getptr(x_), getptr(y_)))) -""" - pyitruediv(x, y) - -In-place true division. `x = pyitruediv(x, y)` is equivalent to `x /= y` in Python. -""" -pyitruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(getptr(x_), getptr(y_)))) -""" - pyimod(x, y) - -In-place subtraction. `x = pyimod(x, y)` is equivalent to `x %= y` in Python. -""" -pyimod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(getptr(x_), getptr(y_)))) -""" - pyilshift(x, y) - -In-place left shift. `x = pyilshift(x, y)` is equivalent to `x <<= y` in Python. -""" -pyilshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(getptr(x_), getptr(y_)))) -""" - pyirshift(x, y) - -In-place right shift. `x = pyirshift(x, y)` is equivalent to `x >>= y` in Python. -""" -pyirshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(getptr(x_), getptr(y_)))) -""" - pyiand(x, y) - -In-place and. `x = pyiand(x, y)` is equivalent to `x &= y` in Python. -""" -pyiand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(getptr(x_), getptr(y_)))) -""" - pyixor(x, y) - -In-place xor. `x = pyixor(x, y)` is equivalent to `x ^= y` in Python. -""" -pyixor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(getptr(x_), getptr(y_)))) -""" - pyior(x, y) - -In-place or. `x = pyior(x, y)` is equivalent to `x |= y` in Python. -""" -pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(getptr(x_), getptr(y_)))) -export pyiadd, pyisub, pyimul, pyimatmul, pyifloordiv, pyitruediv, pyimod, pyilshift, pyirshift, pyiand, pyixor, pyior - -# power -""" - pypow(x, y, z=None) - -Equivalent to `x ** y` or `pow(x, y, z)` in Python. -""" -pypow(x, y, z=pybuiltins.None) = pynew(errcheck(@autopy x y z C.PyNumber_Power(getptr(x_), getptr(y_), getptr(z_)))) -""" - pyipow(x, y, z=None) - -In-place power. `x = pyipow(x, y)` is equivalent to `x **= y` in Python. -""" -pyipow(x, y, z=pybuiltins.None) = pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(getptr(x_), getptr(y_), getptr(z_)))) -export pypow, pyipow diff --git a/src/abstract/object.jl b/src/abstract/object.jl deleted file mode 100644 index 7f404095..00000000 --- a/src/abstract/object.jl +++ /dev/null @@ -1,309 +0,0 @@ -""" - pyis(x, y) - -True if `x` and `y` are the same Python object. Equivalent to `x is y` in Python. -""" -pyis(x, y) = @autopy x y getptr(x_) == getptr(y_) -export pyis - -pyisnot(x, y) = !pyis(x, y) - -""" - pyrepr(x) - -Equivalent to `repr(x)` in Python. -""" -pyrepr(x) = pynew(errcheck(@autopy x C.PyObject_Repr(getptr(x_)))) -pyrepr(::Type{String}, x) = (s=pyrepr(x); ans=pystr_asstring(s); pydel!(s); ans) -export pyrepr - -""" - pyascii(x) - -Equivalent to `ascii(x)` in Python. -""" -pyascii(x) = pynew(errcheck(@autopy x C.PyObject_ASCII(getptr(x_)))) -pyascii(::Type{String}, x) = (s=pyascii(x); ans=pystr_asstring(s); pydel!(s); ans) -export pyascii - -""" - pyhasattr(x, k) - -Equivalent to `hasattr(x, k)` in Python. - -Tests if `getattr(x, k)` raises an `AttributeError`. -""" -function pyhasattr(x, k) - ptr = @autopy x k C.PyObject_GetAttr(getptr(x_), getptr(k_)) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return false - else - pythrow() - end - else - decref(ptr) - return true - end -end -# pyhasattr(x, k) = errcheck(@autopy x k C.PyObject_HasAttr(getptr(x_), getptr(k_))) == 1 -export pyhasattr - -""" - pygetattr(x, k, [d]) - -Equivalent to `getattr(x, k)` or `x.k` in Python. - -If `d` is specified, it is returned if the attribute does not exist. -""" -pygetattr(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetAttr(getptr(x_), getptr(k_)))) -function pygetattr(x, k, d) - ptr = @autopy x k C.PyObject_GetAttr(getptr(x_), getptr(k_)) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return d - else - pythrow() - end - else - return pynew(ptr) - end -end -export pygetattr - -""" - pysetattr(x, k, v) - -Equivalent to `setattr(x, k, v)` or `x.k = v` in Python. -""" -pysetattr(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetAttr(getptr(x_), getptr(k_), getptr(v_))); nothing) -export pysetattr - -""" - pydelattr(x, k) - -Equivalent to `delattr(x, k)` or `del x.k` in Python. -""" -pydelattr(x, k) = (errcheck(@autopy x k C.PyObject_SetAttr(getptr(x_), getptr(k_), C.PyNULL)); nothing) -export pydelattr - -""" - pyissubclass(s, t) - -Test if `s` is a subclass of `t`. Equivalent to `issubclass(s, t)` in Python. -""" -pyissubclass(s, t) = errcheck(@autopy s t C.PyObject_IsSubclass(getptr(s_), getptr(t_))) == 1 -export pyissubclass - -""" - pyisinstance(x, t) - -Test if `x` is of type `t`. Equivalent to `isinstance(x, t)` in Python. -""" -pyisinstance(x, t) = errcheck(@autopy x t C.PyObject_IsInstance(getptr(x_), getptr(t_))) == 1 -export pyisinstance - -""" - pyhash(x) - -Equivalent to `hash(x)` in Python, converted to an `Integer`. -""" -pyhash(x) = errcheck(@autopy x C.PyObject_Hash(getptr(x_))) -export pyhash - -""" - pytruth(x) - -The truthyness of `x`. Equivalent to `bool(x)` in Python, converted to a `Bool`. -""" -pytruth(x) = errcheck(@autopy x C.PyObject_IsTrue(getptr(x_))) == 1 -export pytruth - -""" - pynot(x) - -The falsyness of `x`. Equivalent to `not x` in Python, converted to a `Bool`. -""" -pynot(x) = errcheck(@autopy x C.PyObject_Not(getptr(x_))) == 1 -export pynot - -""" - pylen(x) - -The length of `x`. Equivalent to `len(x)` in Python, converted to an `Integer`. -""" -pylen(x) = errcheck(@autopy x C.PyObject_Length(getptr(x_))) -export pylen - -""" - pyhasitem(x, k) - -Test if `pygetitem(x, k)` raises a `KeyError` or `AttributeError`. -""" -function pyhasitem(x, k) - ptr = @autopy x k C.PyObject_GetItem(getptr(x_), getptr(k_)) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return false - else - pythrow() - end - else - decref(ptr) - return true - end -end -export pyhasitem - -""" - pygetitem(x, k, [d]) - -Equivalent `x[k]` in Python. - -If `d` is specified, it is returned if the item does not exist (i.e. if `x[k]` raises a -`KeyError` or `IndexError`). -""" -pygetitem(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetItem(getptr(x_), getptr(k_)))) -function pygetitem(x, k, d) - ptr = @autopy x k C.PyObject_GetItem(getptr(x_), getptr(k_)) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return d - else - pythrow() - end - else - return pynew(ptr) - end -end -export pygetitem - -""" - pysetitem(x, k, v) - -Equivalent to `setitem(x, k, v)` or `x[k] = v` in Python. -""" -pysetitem(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetItem(getptr(x_), getptr(k_), getptr(v_))); nothing) -export pysetitem - -""" - pydelitem(x, k) - -Equivalent to `delitem(x, k)` or `del x[k]` in Python. -""" -pydelitem(x, k) = (errcheck(@autopy x k C.PyObject_DelItem(getptr(x_), getptr(k_))); nothing) -export pydelitem - -""" - pydir(x) - -Equivalent to `dir(x)` in Python. -""" -pydir(x) = pynew(errcheck(@autopy x C.PyObject_Dir(getptr(x_)))) -export pydir - -pycallargs(f) = pynew(errcheck(@autopy f C.PyObject_CallObject(getptr(f_), C.PyNULL))) -pycallargs(f, args) = pynew(errcheck(@autopy f args C.PyObject_CallObject(getptr(f_), getptr(args_)))) -pycallargs(f, args, kwargs) = pynew(errcheck(@autopy f args kwargs C.PyObject_Call(getptr(f_), getptr(args_), getptr(kwargs_)))) - -""" - pycall(f, args...; kwargs...) - -Call the Python object `f` with the given arguments. -""" -pycall(f, args...; kwargs...) = - if !isempty(kwargs) - args_ = pytuple_fromiter(args) - kwargs_ = pystrdict_fromiter(kwargs) - ans = pycallargs(f, args_, kwargs_) - pydel!(args_) - pydel!(kwargs_) - ans - elseif !isempty(args) - args_ = pytuple_fromiter(args) - ans = pycallargs(f, args_) - pydel!(args_) - ans - else - pycallargs(f) - end -export pycall - -""" - pyeq(x, y) - pyeq(Bool, x, y) - -Equivalent to `x == y` in Python. The second form converts to `Bool`. -""" -pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_EQ))) - -""" - pyne(x, y) - pyne(Bool, x, y) - -Equivalent to `x != y` in Python. The second form converts to `Bool`. -""" -pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_NE))) - -""" - pyle(x, y) - pyle(Bool, x, y) - -Equivalent to `x <= y` in Python. The second form converts to `Bool`. -""" -pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_LE))) - -""" - pylt(x, y) - pylt(Bool, x, y) - -Equivalent to `x < y` in Python. The second form converts to `Bool`. -""" -pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_LT))) - -""" - pyge(x, y) - pyge(Bool, x, y) - -Equivalent to `x >= y` in Python. The second form converts to `Bool`. -""" -pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_GE))) - -""" - pygt(x, y) - pygt(Bool, x, y) - -Equivalent to `x > y` in Python. The second form converts to `Bool`. -""" -pygt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(getptr(x_), getptr(y_), C.Py_GT))) -pyeq(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_EQ)) == 1 -pyne(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_NE)) == 1 -pyle(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_LE)) == 1 -pylt(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_LT)) == 1 -pyge(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_GE)) == 1 -pygt(::Type{Bool}, x, y) = errcheck(@autopy x y C.PyObject_RichCompareBool(getptr(x_), getptr(y_), C.Py_GT)) == 1 -export pyeq, pyne, pyle, pylt, pyge, pygt - -pyconvert_rule_object(::Type{Py}, x::Py) = pyconvert_return(x) - -""" - pycontains(x, v) - -Equivalent to `v in x` in Python. -""" -pycontains(x, v) = errcheck(@autopy x v C.PySequence_Contains(getptr(x_), getptr(v_))) == 1 -export pycontains - -""" - pyin(v, x) - -Equivalent to `v in x` in Python. -""" -pyin(v, x) = pycontains(x, v) -export pyin - -pynotin(v, x) = !pyin(v, x) diff --git a/src/compat/_.jl b/src/compat/_.jl new file mode 100644 index 00000000..3a14e088 --- /dev/null +++ b/src/compat/_.jl @@ -0,0 +1,30 @@ +""" + module _compat + +Misc bits and bobs for compatibility. +""" +module _compat + using .._Py + using .._Py: C, Utils, pynew, incref, getptr, pycopy!, pymodulehooks, pyisnull, pybytes_asvector, pysysmodule, pyosmodule, pystr_fromUTF8 + using .._pyconvert: pyconvert, @pyconvert + using .._pywrap: PyPandasDataFrame + using Serialization: Serialization, AbstractSerializer, serialize, deserialize + using Tables: Tables + using Requires: @require + + include("gui.jl") + include("ipython.jl") + include("multimedia.jl") + include("serialization.jl") + include("tables.jl") + include("pycall.jl") + + function __init__() + C.with_gil() do + init_gui() + init_pyshow() + init_tables() + end + @require PyCall="438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall) + end +end diff --git a/src/compat/gui.jl b/src/compat/gui.jl index 2d654419..f433b2e7 100644 --- a/src/compat/gui.jl +++ b/src/compat/gui.jl @@ -13,12 +13,12 @@ function fix_qt_plugin_path() C.CTX.exe_path === nothing && return false e = pyosmodule.environ "QT_PLUGIN_PATH" in e && return false - qtconf = joinpath(dirname(C.CTX.exe_path), "qt.conf") + qtconf = joinpath(dirname(C.CTX.exe_path::AbstractString), "qt.conf") isfile(qtconf) || return false for line in eachline(qtconf) m = match(r"^\s*prefix\s*=(.*)$"i, line) if m !== nothing - path = strip(m.captures[1]) + path = strip(m.captures[1]::AbstractString) path[1] == path[end] == '"' && (path = path[2:end-1]) path = joinpath(path, "plugins") if isdir(path) @@ -61,8 +61,8 @@ const new_event_loop_callback = pynew() function init_gui() if !C.CTX.is_embedded # define callbacks - @py g = {} - @py @exec """ + g = pydict() + pyexec(""" def new_event_loop_callback(g, interval=0.04): if g in ("pyqt4","pyqt5","pyside","pyside2"): if g == "pyqt4": @@ -126,11 +126,11 @@ function init_gui() else: raise ValueError("invalid event loop name: {}".format(g)) return callback - """ g + """, g) pycopy!(new_event_loop_callback, g["new_event_loop_callback"]) # add a hook to automatically call fix_qt_plugin_path() - fixqthook = Py(() -> (CONFIG.auto_fix_qt_plugin_path && fix_qt_plugin_path(); nothing)) + fixqthook = Py(() -> (_Py.CONFIG.auto_fix_qt_plugin_path && fix_qt_plugin_path(); nothing)) pymodulehooks.add_hook("PyQt4", fixqthook) pymodulehooks.add_hook("PyQt5", fixqthook) pymodulehooks.add_hook("PySide", fixqthook) diff --git a/src/compat/multimedia.jl b/src/compat/multimedia.jl index 024ce9e7..3049f359 100644 --- a/src/compat/multimedia.jl +++ b/src/compat/multimedia.jl @@ -125,3 +125,23 @@ function init_pyshow() pyshow_add_rule(pyshow_rule_repr) pyshow_add_rule(pyshow_rule_savefig) end + +### Py + +Base.show(io::IO, mime::MIME, o::Py) = pyshow(io, mime, o) +Base.show(io::IO, mime::MIME"text/csv", o::Py) = pyshow(io, mime, o) +Base.show(io::IO, mime::MIME"text/tab-separated-values", o::Py) = pyshow(io, mime, o) +Base.showable(mime::MIME, o::Py) = pyshowable(mime, o) + +### PyPandasDataFrame + +function Base.show(io::IO, mime::MIME"text/plain", df::PyPandasDataFrame) + nrows = pyconvert(Int, Py(df).shape[0]) + ncols = pyconvert(Int, Py(df).shape[1]) + printstyled(io, nrows, '×', ncols, ' ', typeof(df), '\n', bold=true) + pyshow(io, mime, df) +end +Base.show(io::IO, mime::MIME, df::PyPandasDataFrame) = pyshow(io, mime, df) +Base.show(io::IO, mime::MIME"text/csv", df::PyPandasDataFrame) = pyshow(io, mime, df) +Base.show(io::IO, mime::MIME"text/tab-separated-values", df::PyPandasDataFrame) = pyshow(io, mime, df) +Base.showable(mime::MIME, df::PyPandasDataFrame) = pyshowable(mime, df) diff --git a/src/compat/pycall.jl b/src/compat/pycall.jl new file mode 100644 index 00000000..0bf9cec7 --- /dev/null +++ b/src/compat/pycall.jl @@ -0,0 +1,21 @@ +function init_pycall(PyCall::Module) + # allow explicit conversion between PythonCall.Py and PyCall.PyObject + # provided they are using the same interpretr + errmsg = """ + Conversion between `PyCall.PyObject` and `PythonCall.Py` is only possible when using the same Python interpreter. + + There are two ways to achieve this: + - Set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`. This forces PythonCall to use the same + interpreter as PyCall, but PythonCall loses the ability to manage its own dependencies. + - Set the environment variable `PYTHON` to `PythonCall.C.CTX.exe_path` and rebuild PyCall. This forces PyCall + to use the same interpreter as PythonCall, but needs to be repeated whenever you switch Julia environment. + """ + @eval function _Py.Py(x::$PyCall.PyObject) + C.CTX.matches_pycall::Bool || error($errmsg) + return pynew(C.PyPtr($PyCall.pyreturn(x))) + end + @eval function PyCall.PyObject(x::Py) + C.CTX.matches_pycall::Bool || error($errmsg) + return $PyCall.PyObject($PyCall.PyPtr(incref(getptr(x)))) + end +end diff --git a/src/compat/tables.jl b/src/compat/tables.jl index 60ad899c..f7cc376f 100644 --- a/src/compat/tables.jl +++ b/src/compat/tables.jl @@ -17,11 +17,11 @@ function pytable(src, format=:pandas; opts...) if format == :pandas _pytable_pandas(src; opts...) elseif format == :columns - _pytable_columns(src; opts...) + _pytable_columns(src) elseif format == :rows - _pytable_rows(src; opts...) + _pytable_rows(src) elseif format == :rowdicts - _pytable_rowdicts(src; opts...) + _pytable_rowdicts(src) else error("invalid format") end diff --git a/src/compat/with.jl b/src/compat/with.jl deleted file mode 100644 index 946f692f..00000000 --- a/src/compat/with.jl +++ /dev/null @@ -1,30 +0,0 @@ -""" - pywith(f, o, d=nothing) - -Equivalent to `with o as x: f(x)` in Python, where `x` is a `Py`. - -On success, the value of `f(x)` is returned. - -If an exception occurs but is suppressed then `d` is returned. -""" -function pywith(f, o, d = nothing) - o = Py(o) - t = pytype(o) - exit = t.__exit__ - value = t.__enter__(o) - exited = false - try - return f(value) - catch exc - if exc isa PyException - exited = true - if pytruth(exit(o, exc.t, exc.v, exc.b)) - return d - end - end - rethrow() - finally - exited || exit(o, pybuiltins.None, pybuiltins.None, pybuiltins.None) - end -end -export pywith diff --git a/src/concrete/bool.jl b/src/concrete/bool.jl deleted file mode 100644 index 1ca712e5..00000000 --- a/src/concrete/bool.jl +++ /dev/null @@ -1,31 +0,0 @@ -""" - pybool(x) - -Convert `x` to a Python `bool`. -""" -pybool(x::Bool=false) = pynew(x ? pybuiltins.True : pybuiltins.False) -pybool(x::Number) = pybool(!iszero(x)) -pybool(x) = pybuiltins.bool(x) -export pybool - -pyisTrue(x) = pyis(x, pybuiltins.True) -pyisFalse(x) = pyis(x, pybuiltins.False) -pyisbool(x) = pyisTrue(x) || pyisFalse(x) - -pybool_asbool(x) = - @autopy x if pyisTrue(x_) - true - elseif pyisFalse(x_) - false - else - error("not a bool") - end - -function pyconvert_rule_bool(::Type{T}, x::Py) where {T<:Number} - val = pybool_asbool(x) - if T in (Bool, Int8, Int16, Int32, Int64, Int128, UInt8, UInt16, UInt32, UInt64, UInt128, BigInt) - pyconvert_return(T(val)) - else - pyconvert_tryconvert(T, val) - end -end diff --git a/src/concrete/bytes.jl b/src/concrete/bytes.jl deleted file mode 100644 index 16fd2aea..00000000 --- a/src/concrete/bytes.jl +++ /dev/null @@ -1,37 +0,0 @@ -pybytes_fromdata(x::Ptr, n::Integer) = pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) -pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) - -""" - pybytes(x) - -Convert `x` to a Python `bytes`. -""" -pybytes(x) = pynew(errcheck(@autopy x C.PyObject_Bytes(getptr(x_)))) -pybytes(x::Vector{UInt8}) = pybytes_fromdata(x) -pybytes(x::Base.CodeUnits{UInt8, String}) = pybytes_fromdata(x) -pybytes(x::Base.CodeUnits{UInt8, SubString{String}}) = pybytes_fromdata(x) -pybytes(::Type{T}, x) where {Vector{UInt8} <: T <: Vector} = (b=pybytes(x); ans=pybytes_asvector(b); pydel!(b); ans) -pybytes(::Type{T}, x) where {Base.CodeUnits{UInt8,String} <: T <: Base.CodeUnits} = (b=pybytes(x); ans=Base.CodeUnits(pybytes_asUTF8string(b)); pydel!(b); ans) -export pybytes - -pyisbytes(x) = pytypecheckfast(x, C.Py_TPFLAGS_BYTES_SUBCLASS) - -function pybytes_asdata(x::Py) - ptr = Ref(Ptr{Cchar}(0)) - len = Ref(C.Py_ssize_t(0)) - Base.GC.@preserve x errcheck(C.PyBytes_AsStringAndSize(getptr(x), ptr, len)) - ptr[], len[] -end - -function pybytes_asvector(x::Py) - ptr, len = pybytes_asdata(x) - unsafe_wrap(Array, Ptr{UInt8}(ptr), len) -end - -function pybytes_asUTF8string(x::Py) - ptr, len = pybytes_asdata(x) - unsafe_string(Ptr{UInt8}(ptr), len) -end - -pyconvert_rule_bytes(::Type{Vector{UInt8}}, x::Py) = pyconvert_return(copy(pybytes_asvector(x))) -pyconvert_rule_bytes(::Type{Base.CodeUnits{UInt8,String}}, x::Py) = pyconvert_return(codeunits(pybytes_asUTF8string(x))) diff --git a/src/concrete/code.jl b/src/concrete/code.jl deleted file mode 100644 index 93c88bd1..00000000 --- a/src/concrete/code.jl +++ /dev/null @@ -1,290 +0,0 @@ -const MODULE_GLOBALS = Dict{Module,Py}() - -function _pyeval_args(code, globals, locals) - if code isa AbstractString - code_ = code - elseif ispy(code) - code_ = code - else - throw(ArgumentError("code must be a string or Python code")) - end - if globals isa Module - globals_ = get!(pydict, MODULE_GLOBALS, globals) - elseif ispy(globals) - globals_ = globals - else - throw(ArgumentError("globals must be a module or a Python dict")) - end - if locals === nothing - locals_ = pynew(Py(globals_)) - elseif ispy(locals) - locals_ = pynew(Py(locals)) - else - locals_ = pydict(locals) - end - return (code_, globals_, locals_) -end - -""" - pyeval([T=Py], code, globals, locals=nothing) - -Evaluate the given Python `code`, returning the result as a `T`. - -If `globals` is a `Module`, then a persistent `dict` unique to that module is used. - -By default the code runs in global scope (i.e. `locals===globals`). To use a temporary -local scope, set `locals` to `()`, or to a `NamedTuple` of variables to include in the -scope. - -See also [`@pyeval`](@ref). - -# Examples - -The following computes `1.1+2.2` in the `Main` module as a `Float64`: -``` -pyeval(Float64, "x+y", Main, (x=1.1, y=2.2)) # returns 3.3 -``` -""" -function pyeval(::Type{T}, code, globals, locals=nothing) where {T} - code_, globals_, locals_ = _pyeval_args(code, globals, locals) - ans = pybuiltins.eval(code_, globals_, locals_) - pydel!(locals_) - return T == Py ? ans : pyconvert(T, ans) -end -pyeval(code, globals, locals=nothing) = pyeval(Py, code, globals, locals) -export pyeval - -_pyexec_ans(::Type{Nothing}, globals, locals) = nothing -@generated function _pyexec_ans(::Type{NamedTuple{names, types}}, globals, locals) where {names, types} - # TODO: use precomputed interned strings - # TODO: try to load from globals too - n = length(names) - code = [] - vars = Symbol[] - for i in 1:n - v = Symbol(:ans, i) - push!(vars, v) - push!(code, :($v = pyconvert($(types.parameters[i]), pygetitem(locals, $(string(names[i])))))) - end - push!(code, :(return $(NamedTuple{names, types})(($(vars...),)))) - return Expr(:block, code...) -end - -""" - pyexec([T=Nothing], code, globals, locals=nothing) - -Execute the given Python `code`. - -If `globals` is a `Module`, then a persistent `dict` unique to that module is used. - -By default the code runs in global scope (i.e. `locals===globals`). To use a temporary -local scope, set `locals` to `()`, or to a `NamedTuple` of variables to include in the -scope. - -If `T==Nothing` then returns `nothing`. Otherwise `T` must be a concrete `NamedTuple` type -and the corresponding items from `locals` are extracted and returned. - -See also [`@pyexec`](@ref). - -# Examples - -The following computes `1.1+2.2` in the `Main` module as a `Float64`: -``` -pyexec(@NamedTuple{ans::Float64}, "ans=x+y", Main, (x=1.1, y=2.2)) # returns (ans = 3.3,) -``` - -Marking variables as `global` saves them into the module scope, so that they are available -in subsequent invocations: -``` -pyexec("global x; x=12", Main) -pyeval(Int, "x", Main) # returns 12 -``` -""" -function pyexec(::Type{T}, code, globals, locals=nothing) where {T} - code_, globals_, locals_ = _pyeval_args(code, globals, locals) - pydel!(pybuiltins.exec(code_, globals_, locals_)) - ans = _pyexec_ans(T, globals_, locals_) - pydel!(locals_) - return ans -end -pyexec(code, globals, locals=nothing) = pyexec(Nothing, code, globals, locals) -export pyexec - -function _pyeval_macro_code(arg) - if arg isa String - return arg - elseif arg isa Expr && arg.head === :macrocall && arg.args[1] == :(`foo`).args[1] - return arg.args[3] - else - return nothing - end -end - -function _pyeval_macro_args(arg, filename, mode) - # separate out inputs => code => outputs (with only code being required) - if @capture(arg, inputs_ => code_ => outputs_) - code = _pyeval_macro_code(code) - code === nothing && error("invalid code") - elseif @capture(arg, lhs_ => rhs_) - code = _pyeval_macro_code(lhs) - if code === nothing - code = _pyeval_macro_code(rhs) - code === nothing && error("invalid code") - inputs = lhs - outputs = nothing - else - inputs = nothing - outputs = rhs - end - else - code = _pyeval_macro_code(arg) - code === nothing && error("invalid code") - inputs = outputs = nothing - end - # precompile the code - codestr = code - codeobj = pynew() - codeready = Ref(false) - code = quote - if !$codeready[] - $pycopy!($codeobj, $pybuiltins.compile($codestr, $filename, $mode)) - $codeready[] = true - end - $codeobj - end - # convert inputs to locals - if inputs === nothing - locals = () - else - if inputs isa Expr && inputs.head === :tuple - inputs = inputs.args - else - inputs = [inputs] - end - locals = [] - for input in inputs - if @capture(input, var_Symbol) - push!(locals, var => var) - elseif @capture(input, var_Symbol = ex_) - push!(locals, var => ex) - else - error("invalid input: $input") - end - end - locals = :(($([:($var = $ex) for (var,ex) in locals]...),)) - end - # done - return locals, code, outputs -end - -""" - @pyeval [inputs =>] code [=> T] - -Evaluate the given `code` in a new local scope and return the answer as a `T`. - -The global scope is persistent and unique to the current module. - -The `code` must be a literal string or command. - -The `inputs` is a tuple of inputs of the form `v=expr` to be included in the local scope. -Only `v` is required, `expr` defaults to `v`. - -# Examples - -The following computes `1.1+2.2` and returns a `Float64`: -``` -@pyeval (x=1.1, y=2.2) => `x+y` => Float64 # returns 3.3 -``` -""" -macro pyeval(arg) - locals, code, outputs = _pyeval_macro_args(arg, "$(__source__.file):$(__source__.line)", "eval") - if outputs === nothing - outputs = Py - end - esc(:($pyeval($outputs, $code, $__module__, $locals))) -end -export @pyeval - -""" - @pyexec [inputs =>] code [=> outputs] - -Execute the given `code` in a new local scope. - -The global scope is persistent and unique to the current module. - -The `code` must be a literal string or command. - -The `inputs` is a tuple of inputs of the form `v=expr` to be included in the local scope. -Only `v` is required, `expr` defaults to `v`. - -The `outputs` is a tuple of outputs of the form `x::T=v`, meaning that `v` is extracted from -locals, converted to `T` and assigned to `x`. Only `x` is required: `T` defaults to `Py` -and `v` defaults to `x`. - -# Examples - -The following computes `1.1+2.2` and assigns its value to `ans` as a `Float64`: -``` -@pyexec (x=1.1, y=2.2) => `ans=x+y` => ans::Float64 # returns 3.3 -``` - -Marking variables as `global` saves them into the module scope, so that they are available -in subsequent invocations: -``` -@pyexec `global x; x=12` -@pyeval `x` => Int # returns 12 -``` -""" -macro pyexec(arg) - locals, code, outputs = _pyeval_macro_args(arg, "$(__source__.file):$(__source__.line)", "exec") - if outputs === nothing - outputs = Nothing - esc(:($pyexec(Nothing, $code, $__module__, $locals))) - else - if outputs isa Expr && outputs.head === :tuple - oneoutput = false - outputs = outputs.args - else - oneoutput = true - outputs = [outputs] - end - pyvars = Symbol[] - jlvars = Symbol[] - types = [] - for output in outputs - if @capture(output, lhs_ = rhs_) - rhs isa Symbol || error("invalid output: $output") - output = lhs - pyvar = rhs - else - pyvar = missing - end - if @capture(output, lhs_ :: rhs_) - outtype = rhs - output = lhs - else - outtype = Py - end - output isa Symbol || error("invalid output: $output") - if pyvar === missing - pyvar = output - end - push!(pyvars, pyvar) - push!(jlvars, output) - push!(types, outtype) - end - outtype = :($NamedTuple{($(map(QuoteNode, pyvars)...),), Tuple{$(types...),}}) - ans = :($pyexec($outtype, $code, $__module__, $locals)) - if oneoutput - ans = :($(jlvars[1]) = $ans[1]) - else - if pyvars != jlvars - outtype2 = :($NamedTuple{($(map(QuoteNode, jlvars)...),), Tuple{$(types...),}}) - ans = :($outtype2($ans)) - end - ans = :(($(jlvars...),) = $ans) - end - esc(ans) - end -end -export @pyexec diff --git a/src/concrete/complex.jl b/src/concrete/complex.jl deleted file mode 100644 index 99a61ca7..00000000 --- a/src/concrete/complex.jl +++ /dev/null @@ -1,33 +0,0 @@ -# :PyComplex_FromDoubles => (Cdouble, Cdouble) => PyPtr, -# :PyComplex_RealAsDouble => (PyPtr,) => Cdouble, -# :PyComplex_ImagAsDouble => (PyPtr,) => Cdouble, -# :PyComplex_AsCComplex => (PyPtr,) => Py_complex, - -""" - pycomplex(x=0.0) - pycomplex(re, im) - -Convert `x` to a Python `complex`, or create one from given real and imaginary parts. -""" -pycomplex(x::Real=0.0, y::Real=0.0) = pynew(errcheck(C.PyComplex_FromDoubles(x, y))) -pycomplex(x::Complex) = pycomplex(real(x), imag(x)) -pycomplex(x) = pybuiltins.complex(x) -pycomplex(x, y) = pybuiltins.complex(x, y) -export pycomplex - -pyiscomplex(x) = pytypecheck(x, pybuiltins.complex) - -function pycomplex_ascomplex(x) - c = @autopy x C.PyComplex_AsCComplex(getptr(x_)) - c.real == -1 && c.imag == 0 && errcheck() - return Complex(c.real, c.imag) -end - -function pyconvert_rule_complex(::Type{T}, x::Py) where {T<:Number} - val = pycomplex_ascomplex(x) - if T in (Complex{Float64}, Complex{Float32}, Complex{Float16}, Complex{BigFloat}) - pyconvert_return(T(val)) - else - pyconvert_tryconvert(T, val) - end -end diff --git a/src/concrete/datetime.jl b/src/concrete/datetime.jl deleted file mode 100644 index 1e0c41a8..00000000 --- a/src/concrete/datetime.jl +++ /dev/null @@ -1,95 +0,0 @@ -# We used to use 1/1/1 but pandas.Timestamp is a subclass of datetime and does not include -# this date, so we use 1970 instead. -const _base_datetime = DateTime(1970, 1, 1) -const _base_pydatetime = pynew() - -function init_datetime() - pycopy!(_base_pydatetime, pydatetimetype(1970, 1, 1)) -end - -pydate(year, month, day) = pydatetype(year, month, day) -pydate(x::Date) = pydate(year(x), month(x), day(x)) -export pydate - -pytime(_hour=0, _minute=0, _second=0, _microsecond=0, _tzinfo=nothing; hour=_hour, minute=_minute, second=_second, microsecond=_microsecond, tzinfo=_tzinfo, fold=0) = pytimetype(hour, minute, second, microsecond, tzinfo, fold=fold) -pytime(x::Time) = - if iszero(nanosecond(x)) - pytime(hour(x), minute(x), second(x), millisecond(x) * 1000 + microsecond(x)) - else - errset(pybuiltins.ValueError, "cannot create 'datetime.time' with less than microsecond resolution") - pythrow() - end -export pytime - -pydatetime(year, month, day, _hour=0, _minute=0, _second=0, _microsecond=0, _tzinfo=nothing; hour=_hour, minute=_minute, second=_second, microsecond=_microsecond, tzinfo=_tzinfo, fold=0) = pydatetimetype(year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold) -function pydatetime(x::DateTime) - # compute time since _base_datetime - # this accounts for fold - d = pytimedeltatype(milliseconds = (x - _base_datetime).value) - ans = _base_pydatetime + d - pydel!(d) - return ans -end -pydatetime(x::Date) = pydatetime(year(x), month(x), day(x)) -export pydatetime - -function pytime_isaware(x) - tzinfo = pygetattr(x, "tzinfo") - if pyisnone(tzinfo) - pydel!(tzinfo) - return false - end - utcoffset = tzinfo.utcoffset - pydel!(tzinfo) - o = utcoffset(nothing) - pydel!(utcoffset) - ans = !pyisnone(o) - pydel!(o) - return ans -end - -function pydatetime_isaware(x) - tzinfo = pygetattr(x, "tzinfo") - if pyisnone(tzinfo) - pydel!(tzinfo) - return false - end - utcoffset = tzinfo.utcoffset - pydel!(tzinfo) - o = utcoffset(x) - pydel!(utcoffset) - ans = !pyisnone(o) - pydel!(o) - return ans -end - -function pyconvert_rule_date(::Type{Date}, x::Py) - # datetime is a subtype of date, but we shouldn't convert datetime to Date since it's lossy - pyisinstance(x, pydatetimetype) && return pyconvert_unconverted() - year = pyconvert(Int, x.year) - month = pyconvert(Int, x.month) - day = pyconvert(Int, x.day) - pyconvert_return(Date(year, month, day)) -end - -function pyconvert_rule_time(::Type{Time}, x::Py) - pytime_isaware(x) && return pyconvert_unconverted() - hour = pyconvert(Int, x.hour) - minute = pyconvert(Int, x.minute) - second = pyconvert(Int, x.second) - microsecond = pyconvert(Int, x.microsecond) - return pyconvert_return(Time(hour, minute, second, div(microsecond, 1000), mod(microsecond, 1000))) -end - -function pyconvert_rule_datetime(::Type{DateTime}, x::Py) - pydatetime_isaware(x) && return pyconvert_unconverted() - # compute the time since _base_datetime - # this accounts for fold - d = x - _base_pydatetime - days = pyconvert(Int, d.days) - seconds = pyconvert(Int, d.seconds) - microseconds = pyconvert(Int, d.microseconds) - pydel!(d) - iszero(mod(microseconds, 1000)) || return pyconvert_unconverted() - return pyconvert_return(_base_datetime + Millisecond(div(microseconds, 1000) + 1000 * (seconds + 60 * 60 * 24 * days))) -end diff --git a/src/concrete/dict.jl b/src/concrete/dict.jl deleted file mode 100644 index e7e4e61f..00000000 --- a/src/concrete/dict.jl +++ /dev/null @@ -1,31 +0,0 @@ -pydict_setitem(x::Py, k, v) = errcheck(@autopy k v C.PyDict_SetItem(getptr(x), getptr(k_), getptr(v_))) - -function pydict_fromiter(kvs) - ans = pydict() - for (k, v) in kvs - pydict_setitem(ans, k, v) - end - return ans -end - -function pystrdict_fromiter(kvs) - ans = pydict() - for (k, v) in kvs - pydict_setitem(ans, string(k), v) - end - return ans -end - -""" - pydict(x) - pydict(; x...) - -Convert `x` to a Python `dict`. In the second form, the keys are strings. - -If `x` is a Python object, this is equivalent to `dict(x)` in Python. -Otherwise `x` must iterate over key-value pairs. -""" -pydict(; kwargs...) = isempty(kwargs) ? pynew(errcheck(C.PyDict_New())) : pystrdict_fromiter(kwargs) -pydict(x) = ispy(x) ? pybuiltins.dict(x) : pydict_fromiter(x) -pydict(x::NamedTuple) = pydict(; x...) -export pydict diff --git a/src/concrete/float.jl b/src/concrete/float.jl deleted file mode 100644 index 303650e0..00000000 --- a/src/concrete/float.jl +++ /dev/null @@ -1,44 +0,0 @@ -# :PyFloat_FromDouble => (Cdouble,) => PyPtr, -# :PyFloat_AsDouble => (PyPtr,) => Cdouble, - -""" - pyfloat(x=0.0) - -Convert `x` to a Python `float`. -""" -pyfloat(x::Real=0.0) = pynew(errcheck(C.PyFloat_FromDouble(x))) -pyfloat(x) = @autopy x pynew(errcheck(C.PyNumber_Float(getptr(x_)))) -export pyfloat - -pyisfloat(x) = pytypecheck(x, pybuiltins.float) - -pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(getptr(x_))) - -function pyconvert_rule_float(::Type{T}, x::Py) where {T<:Number} - val = pyfloat_asdouble(x) - if T in (Float16, Float32, Float64, BigFloat) - pyconvert_return(T(val)) - else - pyconvert_tryconvert(T, val) - end -end - -# NaN is sometimes used to represent missing data of other types -# so we allow converting it to Nothing or Missing -function pyconvert_rule_float(::Type{Nothing}, x::Py) - val = pyfloat_asdouble(x) - if isnan(val) - pyconvert_return(nothing) - else - pyconvert_unconverted() - end -end - -function pyconvert_rule_float(::Type{Missing}, x::Py) - val = pyfloat_asdouble(x) - if isnan(val) - pyconvert_return(missing) - else - pyconvert_unconverted() - end -end diff --git a/src/concrete/fraction.jl b/src/concrete/fraction.jl deleted file mode 100644 index 5dba46f8..00000000 --- a/src/concrete/fraction.jl +++ /dev/null @@ -1,19 +0,0 @@ -pyfraction(x::Rational) = pyfraction(numerator(x), denominator(x)) -pyfraction(x, y) = pyfractiontype(x, y) -pyfraction(x) = pyfractiontype(x) -pyfraction() = pyfractiontype() -export pyfraction - -# works for any collections.abc.Rational -function pyconvert_rule_fraction(::Type{R}, x::Py, ::Type{Rational{T0}}=Utils._type_lb(R), ::Type{Rational{T1}}=Utils._type_ub(R)) where {R<:Rational,T0,T1} - a = @pyconvert(Utils._typeintersect(Integer, T1), x.numerator) - b = @pyconvert(Utils._typeintersect(Integer, T1), x.denominator) - a, b = promote(a, b) - T2 = Utils._promote_type_bounded(T0, typeof(a), typeof(b), T1) - pyconvert_return(Rational{T2}(a, b)) -end - -# works for any collections.abc.Rational -function pyconvert_rule_fraction(::Type{T}, x::Py) where {T<:Number} - pyconvert_tryconvert(T, @pyconvert(Rational{<:Integer}, x)) -end diff --git a/src/concrete/import.jl b/src/concrete/import.jl deleted file mode 100644 index 0d86f387..00000000 --- a/src/concrete/import.jl +++ /dev/null @@ -1,15 +0,0 @@ -""" - pyimport(m) - pyimport(m => k) - pyimport(m => (k1, k2, ...)) - pyimport(m1, m2, ...) - -Import a module `m`, or an attribute `k`, or a tuple of attributes. - -If several arguments are given, return the results of importing each one in a tuple. -""" -pyimport(m) = pynew(errcheck(@autopy m C.PyImport_Import(getptr(m_)))) -pyimport((m,k)::Pair) = (m_=pyimport(m); k_=pygetattr(m_,k); pydel!(m_); k_) -pyimport((m,ks)::Pair{<:Any,<:Tuple}) = (m_=pyimport(m); ks_=map(k->pygetattr(m_,k), ks); pydel!(m_); ks_) -pyimport(m1, m2, ms...) = map(pyimport, (m1, m2, ms...)) -export pyimport diff --git a/src/concrete/int.jl b/src/concrete/int.jl deleted file mode 100644 index 0317d4da..00000000 --- a/src/concrete/int.jl +++ /dev/null @@ -1,75 +0,0 @@ -# :PyLong_FromLongLong => (Clonglong,) => PyPtr, -# :PyLong_FromUnsignedLongLong => (Culonglong,) => PyPtr, -# :PyLong_FromString => (Ptr{Cchar}, Ptr{Ptr{Cchar}}, Cint) => PyPtr, -# :PyLong_AsLongLong => (PyPtr,) => Clonglong, -# :PyLong_AsUnsignedLongLong => (PyPtr,) => Culonglong, - -pyint_fallback(x::Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UInt64,UInt128,BigInt}) = - pynew(errcheck(C.PyLong_FromString(string(x, base=32), C_NULL, 32))) -pyint_fallback(x::Integer) = pyint_fallback(BigInt(x)) - -""" - pyint(x=0) - -Convert `x` to a Python `int`. -""" -function pyint(x::Integer=0) - y = mod(x, Clonglong) - if x == y - pynew(errcheck(C.PyLong_FromLongLong(y))) - else - pyint_fallback(x) - end -end -function pyint(x::Unsigned) - y = mod(x, Culonglong) - if x == y - pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) - else - pyint_fallback(x) - end -end -pyint(x) = @autopy x pynew(errcheck(C.PyNumber_Long(getptr(x_)))) -export pyint - -pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) - -pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = begin - # first try to convert to Clonglong (or Culonglong if unsigned) - v = T <: Unsigned ? C.PyLong_AsUnsignedLongLong(getptr(x)) : C.PyLong_AsLongLong(getptr(x)) - if !iserrset_ambig(v) - # success - return pyconvert_tryconvert(T, v) - elseif errmatches(pybuiltins.OverflowError) - # overflows Clonglong or Culonglong - errclear() - if T in ( - Bool, - Int8, - Int16, - Int32, - Int64, - Int128, - UInt8, - UInt16, - UInt32, - UInt64, - UInt128, - ) && - typemin(typeof(v)) ≤ typemin(T) && - typemax(T) ≤ typemax(typeof(v)) - # definitely overflows S, give up now - return pyconvert_unconverted() - else - # try converting -> int -> str -> BigInt -> T - x_int = pyint(x) - x_str = pystr(String, x_int) - pydel!(x_int) - v = parse(BigInt, x_str) - return pyconvert_tryconvert(T, v) - end - else - # other error - pythrow() - end -end diff --git a/src/concrete/list.jl b/src/concrete/list.jl deleted file mode 100644 index 403751d2..00000000 --- a/src/concrete/list.jl +++ /dev/null @@ -1,79 +0,0 @@ -pynulllist(len) = pynew(errcheck(C.PyList_New(len))) - -function pylist_setitem(xs::Py, i, x) - errcheck(C.PyList_SetItem(getptr(xs), i, incref(getptr(Py(x))))) - return xs -end - -pylist_append(xs::Py, x) = errcheck(@autopy x C.PyList_Append(getptr(xs), getptr(x_))) - -pylist_astuple(x) = pynew(errcheck(@autopy x C.PyList_AsTuple(getptr(x_)))) - -function pylist_fromiter(xs) - sz = Base.IteratorSize(typeof(xs)) - if sz isa Base.HasLength || sz isa Base.HasShape - # length known - ans = pynulllist(length(xs)) - for (i, x) in enumerate(xs) - pylist_setitem(ans, i-1, x) - end - return ans - else - # length unknown - ans = pynulllist(0) - for x in xs - pylist_append(ans, x) - end - return ans - end -end - -""" - pylist(x=()) - -Convert `x` to a Python `list`. - -If `x` is a Python object, this is equivalent to `list(x)` in Python. -Otherwise `x` must be iterable. -""" -pylist() = pynulllist(0) -pylist(x) = ispy(x) ? pybuiltins.list(x) : pylist_fromiter(x) -export pylist - -""" - pycollist(x::AbstractArray) - -Create a nested Python `list`-of-`list`s from the elements of `x`. For matrices, this is a list of columns. -""" -function pycollist(x::AbstractArray{T,N}) where {T,N} - N == 0 && return pynew(Py(x[])) - d = N - ax = axes(x, d) - ans = pynulllist(length(ax)) - for (i, j) in enumerate(ax) - y = pycollist(selectdim(x, d, j)) - pylist_setitem(ans, i-1, y) - pydel!(y) - end - return ans -end -export pycollist - -""" - pyrowlist(x::AbstractArray) - -Create a nested Python `list`-of-`list`s from the elements of `x`. For matrices, this is a list of rows. -""" -function pyrowlist(x::AbstractArray{T,N}) where {T,N} - ndims(x) == 0 && return pynew(Py(x[])) - d = 1 - ax = axes(x, d) - ans = pynulllist(length(ax)) - for (i, j) in enumerate(ax) - y = pyrowlist(selectdim(x, d, j)) - pylist_setitem(ans, i-1, y) - pydel!(y) - end - return ans -end -export pyrowlist diff --git a/src/concrete/none.jl b/src/concrete/none.jl deleted file mode 100644 index 31accf5b..00000000 --- a/src/concrete/none.jl +++ /dev/null @@ -1,4 +0,0 @@ -pyisnone(x) = pyis(x, pybuiltins.None) - -pyconvert_rule_none(::Type{Nothing}, x::Py) = pyconvert_return(nothing) -pyconvert_rule_none(::Type{Missing}, x::Py) = pyconvert_return(missing) diff --git a/src/concrete/range.jl b/src/concrete/range.jl deleted file mode 100644 index 70bf6256..00000000 --- a/src/concrete/range.jl +++ /dev/null @@ -1,33 +0,0 @@ -""" - pyrange([[start], [stop]], [step]) - -Construct a Python `range`. Unspecified arguments default to `None`. -""" -pyrange(x, y, z) = pybuiltins.range(x, y, z) -pyrange(x, y) = pybuiltins.range(x, y) -pyrange(y) = pybuiltins.range(y) -export pyrange - -pyrange_fromrange(x::AbstractRange) = pyrange(first(x), last(x) + sign(step(x)), step(x)) - -pyisrange(x) = pytypecheck(x, pybuiltins.range) - -function pyconvert_rule_range(::Type{R}, x::Py, ::Type{StepRange{T0,S0}}=Utils._type_lb(R), ::Type{StepRange{T1,S1}}=Utils._type_ub(R)) where {R<:StepRange,T0,S0,T1,S1} - a = @pyconvert(Utils._typeintersect(Integer, T1), x.start) - b = @pyconvert(Utils._typeintersect(Integer, S1), x.step) - c = @pyconvert(Utils._typeintersect(Integer, T1), x.stop) - a′, c′ = promote(a, c - oftype(c, sign(b))) - T2 = Utils._promote_type_bounded(T0, typeof(a′), typeof(c′), T1) - S2 = Utils._promote_type_bounded(S0, typeof(c′), S1) - pyconvert_return(StepRange{T2, S2}(a′, b, c′)) -end - -function pyconvert_rule_range(::Type{R}, x::Py, ::Type{UnitRange{T0}}=Utils._type_lb(R), ::Type{UnitRange{T1}}=Utils._type_ub(R)) where {R<:UnitRange,T0,T1} - b = @pyconvert(Int, x.step) - b == 1 || return pyconvert_unconverted() - a = @pyconvert(Utils._typeintersect(Integer, T1), x.start) - c = @pyconvert(Utils._typeintersect(Integer, T1), x.stop) - a′, c′ = promote(a, c - oftype(c, 1)) - T2 = Utils._promote_type_bounded(T0, typeof(a′), typeof(c′), T1) - pyconvert_return(UnitRange{T2}(a′, c′)) -end diff --git a/src/concrete/set.jl b/src/concrete/set.jl deleted file mode 100644 index 7687ad8a..00000000 --- a/src/concrete/set.jl +++ /dev/null @@ -1,38 +0,0 @@ -# :PySet_New => (PyPtr,) => PyPtr, -# :PyFrozenSet_New => (PyPtr,) => PyPtr, -# :PySet_Add => (PyPtr, PyPtr) => Cint, - -pyset_add(set::Py, x) = (errcheck(@autopy x C.PySet_Add(getptr(set), getptr(x_))); set) - -function pyset_update_fromiter(set::Py, xs) - for x in xs - pyset_add(set, x) - end - return set -end -pyset_fromiter(xs) = pyset_update_fromiter(pyset(), xs) -pyfrozenset_fromiter(xs) = pyset_update_fromiter(pyfrozenset(), xs) - -""" - pyset(x=()) - -Convert `x` to a Python `set`. - -If `x` is a Python object, this is equivalent to `set(x)` in Python. -Otherwise `x` must be iterable. -""" -pyset() = pynew(errcheck(C.PySet_New(C.PyNULL))) -pyset(x) = ispy(x) ? pybuiltins.set(x) : pyset_fromiter(x) -export pyset - -""" - pyfrozenset(x=()) - -Convert `x` to a Python `frozenset`. - -If `x` is a Python object, this is equivalent to `frozenset(x)` in Python. -Otherwise `x` must be iterable. -""" -pyfrozenset() = pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) -pyfrozenset(x) = ispy(x) ? pybuiltins.frozenset(x) : pyfrozenset_fromiter(x) -export pyfrozenset diff --git a/src/concrete/slice.jl b/src/concrete/slice.jl deleted file mode 100644 index d642ff5b..00000000 --- a/src/concrete/slice.jl +++ /dev/null @@ -1,12 +0,0 @@ -# :PySlice_New => (PyPtr, PyPtr, PyPtr) => PyPtr, - -""" - pyslice([start], stop, [step]) - -Construct a Python `slice`. Unspecified arguments default to `None`. -""" -pyslice(x, y, z=pybuiltins.None) = pynew(errcheck(@autopy x y z C.PySlice_New(getptr(x_), getptr(y_), getptr(z_)))) -pyslice(y) = pyslice(pybuiltins.None, y, pybuiltins.None) -export pyslice - -pyisslice(x) = pytypecheck(x, pybuiltins.slice) diff --git a/src/concrete/str.jl b/src/concrete/str.jl deleted file mode 100644 index dc6a394a..00000000 --- a/src/concrete/str.jl +++ /dev/null @@ -1,37 +0,0 @@ -pystr_fromUTF8(x::Ptr, n::Integer) = pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) -pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) - -""" - pystr(x) - -Convert `x` to a Python `str`. -""" -pystr(x) = pynew(errcheck(@autopy x C.PyObject_Str(getptr(x_)))) -pystr(x::String) = pystr_fromUTF8(x) -pystr(x::SubString{String}) = pystr_fromUTF8(x) -pystr(x::Char) = pystr(string(x)) -pystr(::Type{String}, x) = (s=pystr(x); ans=pystr_asstring(s); pydel!(s); ans) -export pystr - -pystr_asUTF8bytes(x::Py) = Base.GC.@preserve x pynew(errcheck(C.PyUnicode_AsUTF8String(getptr(x)))) -pystr_asUTF8vector(x::Py) = (b=pystr_asUTF8bytes(x); ans=pybytes_asvector(b); pydel!(b); ans) -pystr_asstring(x::Py) = (b=pystr_asUTF8bytes(x); ans=pybytes_asUTF8string(b); pydel!(b); ans) - -function pystr_intern!(x::Py) - ptr = Ref(getptr(x)) - C.PyUnicode_InternInPlace(ptr) - setptr!(x, ptr[]) -end - -pyconvert_rule_str(::Type{String}, x::Py) = pyconvert_return(pystr_asstring(x)) -pyconvert_rule_str(::Type{Symbol}, x::Py) = pyconvert_return(Symbol(pystr_asstring(x))) -pyconvert_rule_str(::Type{Char}, x::Py) = begin - s = pystr_asstring(x) - if length(s) == 1 - pyconvert_return(first(s)) - else - pyconvert_unconverted() - end -end - -pyisstr(x) = pytypecheckfast(x, C.Py_TPFLAGS_UNICODE_SUBCLASS) diff --git a/src/concrete/tuple.jl b/src/concrete/tuple.jl deleted file mode 100644 index 56872873..00000000 --- a/src/concrete/tuple.jl +++ /dev/null @@ -1,53 +0,0 @@ -pynulltuple(len) = pynew(errcheck(C.PyTuple_New(len))) - -function pytuple_setitem(xs::Py, i, x) - errcheck(C.PyTuple_SetItem(getptr(xs), i, incref(getptr(Py(x))))) - return xs -end - -function pytuple_getitem(xs::Py, i) - Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(getptr(xs), i)))) -end - -function pytuple_fromiter(xs) - sz = Base.IteratorSize(typeof(xs)) - if sz isa Base.HasLength || sz isa Base.HasShape - # length known, e.g. Tuple, Pair, Vector - ans = pynulltuple(length(xs)) - for (i, x) in enumerate(xs) - pytuple_setitem(ans, i-1, x) - end - return ans - else - # length unknown - xs_ = pylist_fromiter(xs) - ans = pylist_astuple(xs_) - pydel!(xs_) - return ans - end -end - -@generated function pytuple_fromiter(xs::Tuple) - n = length(xs.parameters) - code = [] - push!(code, :(ans = pynulltuple($n))) - for i in 1:n - push!(code, :(pytuple_setitem(ans, $(i-1), xs[$i]))) - end - push!(code, :(return ans)) - return Expr(:block, code...) -end - -""" - pytuple(x=()) - -Convert `x` to a Python `tuple`. - -If `x` is a Python object, this is equivalent to `tuple(x)` in Python. -Otherwise `x` must be iterable. -""" -pytuple() = pynulltuple(0) -pytuple(x) = ispy(x) ? pybuiltins.tuple(x) : pytuple_fromiter(x) -export pytuple - -pyistuple(x) = pytypecheckfast(x, C.Py_TPFLAGS_TUPLE_SUBCLASS) diff --git a/src/concrete/type.jl b/src/concrete/type.jl deleted file mode 100644 index 6cf3a3a6..00000000 --- a/src/concrete/type.jl +++ /dev/null @@ -1,83 +0,0 @@ -""" - pytype(x) - -The Python `type` of `x`. -""" -pytype(x) = pynew(errcheck(@autopy x C.PyObject_Type(getptr(x_)))) -export pytype - -""" - pytype(name, bases, dict) - -Create a new type. Equivalent to `type(name, bases, dict)` in Python. - -If `bases` is not a Python object, it is converted to one using `pytuple`. - -The `dict` may either by a Python object or a Julia iterable. In the latter case, each item -may either be a `name => value` pair or a Python object with a `__name__` attribute. - -In order to use a Julia `Function` as an instance method, it must be wrapped into a Python -function with [`pyfunc`](@ref). Similarly, see also [`pyclassmethod`](@ref), -[`pystaticmethod`](@ref) or [`pyproperty`](@ref). In all these cases, the arguments passed -to the function always have type `Py`. See the example below. - -# Example - -```julia -Foo = pytype("Foo", (), [ - "__module__" => "__main__", - - pyfunc( - name = "__init__", - doc = \"\"\" - Specify x and y to store in the Foo. - - If omitted, y defaults to None. - \"\"\", - function (self, x, y = nothing) - self.x = x - self.y = y - return - end, - ), - - pyfunc( - name = "__repr__", - self -> "Foo(\$(self.x), \$(self.y))", - ), - - pyclassmethod( - name = "frompair", - doc = "Construct a Foo from a tuple of length two.", - (cls, xy) -> cls(xy...), - ), - - pystaticmethod( - name = "hello", - doc = "Prints a friendly greeting.", - (name) -> println("Hello, \$name"), - ), - - "xy" => pyproperty( - doc = "A tuple of x and y.", - get = (self) -> (self.x, self.y), - set = function (self, xy) - (x, y) = xy - self.x = x - self.y = y - nothing - end, - ), -]) -``` -""" -function pytype(name, bases, dict) - bases2 = ispy(bases) ? bases : pytuple(bases) - dict2 = ispy(dict) ? dict : pydict(ispy(item) ? (pygetattr(item, "__name__") => item) : item for item in dict) - pybuiltins.type(name, bases2, dict2) -end - -pyistype(x) = pytypecheckfast(x, C.Py_TPFLAGS_TYPE_SUBCLASS) - -pytypecheck(x, t) = (@autopy x t C.Py_TypeCheck(getptr(x_), getptr(t_))) == 1 -pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(getptr(x_), f)) == 1 diff --git a/src/cpython/CPython.jl b/src/cpython/CPython.jl deleted file mode 100644 index 2cb671f6..00000000 --- a/src/cpython/CPython.jl +++ /dev/null @@ -1,27 +0,0 @@ -""" - module CPython - -This module provides a direct interface to the Python C API. -""" -module C - -import Base: @kwdef -import CondaPkg -import Pkg -using Libdl, Requires, UnsafePointers, Serialization, ..Utils - -include("consts.jl") -include("pointers.jl") -include("extras.jl") -include("context.jl") -include("gil.jl") -include("jlwrap.jl") - -function __init__() - init_context() - with_gil() do - init_jlwrap() - end -end - -end diff --git a/src/cpython/jlwrap.jl b/src/cpython/jlwrap.jl deleted file mode 100644 index a89ebe07..00000000 --- a/src/cpython/jlwrap.jl +++ /dev/null @@ -1,332 +0,0 @@ -# we store the actual julia values here -# the `value` field of `PyJuliaValueObject` indexes into here -const PYJLVALUES = [] -# unused indices in PYJLVALUES -const PYJLFREEVALUES = Int[] - -function _pyjl_new(t::PyPtr, ::PyPtr, ::PyPtr) - o = ccall(UnsafePtr{PyTypeObject}(t).alloc[!], PyPtr, (PyPtr, Py_ssize_t), t, 0) - o == PyNULL && return PyNULL - UnsafePtr{PyJuliaValueObject}(o).weaklist[] = PyNULL - UnsafePtr{PyJuliaValueObject}(o).value[] = 0 - return o -end - -function _pyjl_dealloc(o::PyPtr) - idx = UnsafePtr{PyJuliaValueObject}(o).value[] - if idx != 0 - PYJLVALUES[idx] = nothing - push!(PYJLFREEVALUES, idx) - end - UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == PyNULL || PyObject_ClearWeakRefs(o) - ccall(UnsafePtr{PyTypeObject}(Py_Type(o)).free[!], Cvoid, (PyPtr,), o) - nothing -end - -const PYJLMETHODS = Vector{Any}() - -function PyJulia_MethodNum(f) - @nospecialize f - push!(PYJLMETHODS, f) - return length(PYJLMETHODS) -end - -function _pyjl_isnull(o::PyPtr, ::PyPtr) - ans = PyJuliaValue_IsNull(o) ? POINTERS._Py_TrueStruct : POINTERS._Py_FalseStruct - Py_IncRef(ans) - ans -end - -function _pyjl_callmethod(o::PyPtr, args::PyPtr) - nargs = PyTuple_Size(args) - @assert nargs > 0 - num = PyLong_AsLongLong(PyTuple_GetItem(args, 0)) - num == -1 && return PyNULL - f = PYJLMETHODS[num] - # this form gets defined in jlwrap/base.jl - return _pyjl_callmethod(f, o, args, nargs)::PyPtr -end - -const PYJLBUFCACHE = Dict{Ptr{Cvoid},Any}() - -@kwdef struct PyBufferInfo{N} - # data - ptr::Ptr{Cvoid} - readonly::Bool - # items - itemsize::Int - format::String - # layout - shape::NTuple{N,Int} - strides::NTuple{N,Int} - suboffsets::NTuple{N,Int} = ntuple(i -> -1, N) -end - -_pyjl_get_buffer_impl(obj::PyPtr, buf::Ptr{Py_buffer}, flags::Cint, x, f) = _pyjl_get_buffer_impl(obj, buf, flags, f(x)::PyBufferInfo) - -function _pyjl_get_buffer_impl(obj::PyPtr, buf::Ptr{Py_buffer}, flags::Cint, info::PyBufferInfo{N}) where {N} - b = UnsafePtr(buf) - c = [] - - # not influenced by flags: obj, buf, len, itemsize, ndim - b.obj[] = C_NULL - b.buf[] = info.ptr - b.itemsize[] = info.itemsize - b.len[] = info.itemsize * prod(info.shape) - b.ndim[] = N - - # readonly - if !info.readonly - b.readonly[] = 0 - elseif Utils.isflagset(flags, PyBUF_WRITABLE) - PyErr_SetString(POINTERS.PyExc_BufferError, "not writable") - return Cint(-1) - else - b.readonly[] = 1 - end - - # format - if Utils.isflagset(flags, PyBUF_FORMAT) - push!(c, info.format) - b.format[] = pointer(info.format) - else - b.format[] = C_NULL - end - - # shape - if Utils.isflagset(flags, PyBUF_ND) - shape = Py_ssize_t[info.shape...] - push!(c, shape) - b.shape[] = pointer(shape) - else - b.shape[] = C_NULL - end - - # strides - if Utils.isflagset(flags, PyBUF_STRIDES) - strides = Py_ssize_t[info.strides...] - push!(c, strides) - b.strides[] = pointer(strides) - elseif Utils.size_to_cstrides(info.itemsize, info.shape) == info.strides - b.strides[] = C_NULL - else - PyErr_SetString(POINTERS.PyExc_BufferError, "not C contiguous and strides not requested") - return Cint(-1) - end - - # suboffsets - if all(==(-1), info.suboffsets) - b.suboffsets[] = C_NULL - elseif Utils.isflagset(flags, PyBUF_INDIRECT) - suboffsets = Py_ssize_t[info.suboffsets...] - push!(c, suboffsets) - b.suboffsets[] = pointer(suboffsets) - else - PyErr_SetString(POINTERS.PyExc_BufferError, "indirect array and suboffsets not requested") - return Cint(-1) - end - - # check contiguity - if Utils.isflagset(flags, PyBUF_C_CONTIGUOUS) - if Utils.size_to_cstrides(info.itemsize, info.shape) != info.strides - PyErr_SetString(POINTERS.PyExc_BufferError, "not C contiguous") - return Cint(-1) - end - end - if Utils.isflagset(flags, PyBUF_F_CONTIGUOUS) - if Utils.size_to_fstrides(info.itemsize, info.shape) != info.strides - PyErr_SetString(POINTERS.PyExc_BufferError, "not Fortran contiguous") - return Cint(-1) - end - end - if Utils.isflagset(flags, PyBUF_ANY_CONTIGUOUS) - if Utils.size_to_cstrides(info.itemsize, info.shape) != info.strides && - Utils.size_to_fstrides(info.itemsize, info.shape) != info.strides - PyErr_SetString(POINTERS.PyExc_BufferError, "not contiguous") - return Cint(-1) - end - end - - # internal - cptr = Base.pointer_from_objref(c) - PYJLBUFCACHE[cptr] = c - b.internal[] = cptr - - # obj - Py_IncRef(obj) - b.obj[] = obj - Cint(0) -end - -function _pyjl_get_buffer(o::PyPtr, buf::Ptr{Py_buffer}, flags::Cint) - num_ = PyObject_GetAttrString(o, "_jl_buffer_info") - num_ == C_NULL && (PyErr_Clear(); PyErr_SetString(POINTERS.PyExc_BufferError, "not a buffer"); return Cint(-1)) - num = PyLong_AsLongLong(num_) - Py_DecRef(num_) - num == -1 && return Cint(-1) - try - f = PYJLMETHODS[num] - x = PyJuliaValue_GetValue(o) - return _pyjl_get_buffer_impl(o, buf, flags, x, f)::Cint - catch exc - @debug "error getting the buffer" exc - PyErr_SetString(POINTERS.PyExc_BufferError, "some error occurred getting the buffer") - return Cint(-1) - end -end - -function _pyjl_release_buffer(xo::PyPtr, buf::Ptr{Py_buffer}) - delete!(PYJLBUFCACHE, UnsafePtr(buf).internal[!]) - nothing -end - -function _pyjl_reduce(self::PyPtr, ::PyPtr) - v = _pyjl_serialize(self, PyNULL) - v == PyNULL && return PyNULL - args = PyTuple_New(1) - args == PyNULL && (Py_DecRef(v); return PyNULL) - err = PyTuple_SetItem(args, 0, v) - err == -1 && (Py_DecRef(args); return PyNULL) - red = PyTuple_New(2) - red == PyNULL && (Py_DecRef(args); return PyNULL) - err = PyTuple_SetItem(red, 1, args) - err == -1 && (Py_DecRef(red); return PyNULL) - f = PyObject_GetAttrString(self, "_jl_deserialize") - f == PyNULL && (Py_DecRef(red); return PyNULL) - err = PyTuple_SetItem(red, 0, f) - err == -1 && (Py_DecRef(red); return PyNULL) - return red -end - -function _pyjl_serialize(self::PyPtr, ::PyPtr) - try - io = IOBuffer() - serialize(io, PyJuliaValue_GetValue(self)) - b = take!(io) - return PyBytes_FromStringAndSize(pointer(b), sizeof(b)) - catch e - PyErr_SetString(POINTERS.PyExc_Exception, "error serializing this value") - # wrap sprint in another try-catch block to prevent this function from throwing - try - @debug "Caught exception $(sprint(showerror, e, catch_backtrace()))" - catch - end - return PyNULL - end -end - -function _pyjl_deserialize(t::PyPtr, v::PyPtr) - try - ptr = Ref{Ptr{Cchar}}() - len = Ref{Py_ssize_t}() - err = PyBytes_AsStringAndSize(v, ptr, len) - err == -1 && return PyNULL - io = IOBuffer(unsafe_wrap(Array, Ptr{UInt8}(ptr[]), Int(len[]))) - x = deserialize(io) - return PyJuliaValue_New(t, x) - catch e - PyErr_SetString(POINTERS.PyExc_Exception, "error deserializing this value") - # wrap sprint in another try-catch block to prevent this function from throwing - try - @debug "Caught exception $(sprint(showerror, e, catch_backtrace()))" - catch - end - return PyNULL - end -end - -const _pyjlbase_name = "juliacall.ValueBase" -const _pyjlbase_type = fill(C.PyTypeObject()) -const _pyjlbase_isnull_name = "_jl_isnull" -const _pyjlbase_callmethod_name = "_jl_callmethod" -const _pyjlbase_reduce_name = "__reduce__" -const _pyjlbase_serialize_name = "_jl_serialize" -const _pyjlbase_deserialize_name = "_jl_deserialize" -const _pyjlbase_methods = Vector{PyMethodDef}() -const _pyjlbase_as_buffer = fill(PyBufferProcs()) - -function init_jlwrap() - empty!(_pyjlbase_methods) - push!(_pyjlbase_methods, - PyMethodDef( - name = pointer(_pyjlbase_callmethod_name), - meth = @cfunction(_pyjl_callmethod, PyPtr, (PyPtr, PyPtr)), - flags = Py_METH_VARARGS, - ), - PyMethodDef( - name = pointer(_pyjlbase_isnull_name), - meth = @cfunction(_pyjl_isnull, PyPtr, (PyPtr, PyPtr)), - flags = Py_METH_NOARGS, - ), - PyMethodDef( - name = pointer(_pyjlbase_reduce_name), - meth = @cfunction(_pyjl_reduce, PyPtr, (PyPtr, PyPtr)), - flags = Py_METH_NOARGS, - ), - PyMethodDef( - name = pointer(_pyjlbase_serialize_name), - meth = @cfunction(_pyjl_serialize, PyPtr, (PyPtr, PyPtr)), - flags = Py_METH_NOARGS, - ), - PyMethodDef( - name = pointer(_pyjlbase_deserialize_name), - meth = @cfunction(_pyjl_deserialize, PyPtr, (PyPtr, PyPtr)), - flags = Py_METH_O | Py_METH_CLASS, - ), - PyMethodDef(), - ) - _pyjlbase_as_buffer[] = PyBufferProcs( - get = @cfunction(_pyjl_get_buffer, Cint, (PyPtr, Ptr{Py_buffer}, Cint)), - release = @cfunction(_pyjl_release_buffer, Cvoid, (PyPtr, Ptr{Py_buffer})), - ) - _pyjlbase_type[] = PyTypeObject( - name = pointer(_pyjlbase_name), - basicsize = sizeof(PyJuliaValueObject), - # new = POINTERS.PyType_GenericNew, - new = @cfunction(_pyjl_new, PyPtr, (PyPtr, PyPtr, PyPtr)), - dealloc = @cfunction(_pyjl_dealloc, Cvoid, (PyPtr,)), - flags = Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VERSION_TAG, - weaklistoffset = fieldoffset(PyJuliaValueObject, 3), - # getattro = POINTERS.PyObject_GenericGetAttr, - # setattro = POINTERS.PyObject_GenericSetAttr, - methods = pointer(_pyjlbase_methods), - as_buffer = pointer(_pyjlbase_as_buffer), - ) - o = POINTERS.PyJuliaBase_Type = PyPtr(pointer(_pyjlbase_type)) - if PyType_Ready(o) == -1 - PyErr_Print() - error("Error initializing 'juliacall.ValueBase'") - end -end - -PyJuliaValue_IsNull(o::PyPtr) = UnsafePtr{PyJuliaValueObject}(o).value[] == 0 - -PyJuliaValue_GetValue(o::PyPtr) = PYJLVALUES[UnsafePtr{PyJuliaValueObject}(o).value[]] - -PyJuliaValue_SetValue(o::PyPtr, @nospecialize(v)) = begin - idx = UnsafePtr{PyJuliaValueObject}(o).value[] - if idx == 0 - if isempty(PYJLFREEVALUES) - push!(PYJLVALUES, v) - idx = length(PYJLVALUES) - else - idx = pop!(PYJLFREEVALUES) - PYJLVALUES[idx] = v - end - UnsafePtr{PyJuliaValueObject}(o).value[] = idx - else - PYJLVALUES[idx] = v - end - nothing -end - -PyJuliaValue_New(t::PyPtr, @nospecialize(v)) = begin - if PyType_IsSubtype(t, POINTERS.PyJuliaBase_Type) != 1 - PyErr_SetString(POINTERS.PyExc_TypeError, "Expecting a subtype of 'juliacall.ValueBase'") - return PyNULL - end - o = PyObject_CallObject(t, PyNULL) - o == PyNULL && return PyNULL - PyJuliaValue_SetValue(o, v) - return o -end diff --git a/src/jlwrap/C.jl b/src/jlwrap/C.jl new file mode 100644 index 00000000..a948e21a --- /dev/null +++ b/src/jlwrap/C.jl @@ -0,0 +1,356 @@ +module Cjl + +using ..._CPython: _CPython as C +using ..._Utils: _Utils as Utils +using Base: @kwdef +using UnsafePointers: UnsafePtr +using Serialization: serialize, deserialize + +@kwdef struct PyJuliaValueObject + ob_base::C.PyObject = C.PyObject() + value::Int = 0 + weaklist::C.PyPtr = C_NULL +end + +const PyJuliaBase_Type = Ref(C.PyNULL) + +# we store the actual julia values here +# the `value` field of `PyJuliaValueObject` indexes into here +const PYJLVALUES = [] +# unused indices in PYJLVALUES +const PYJLFREEVALUES = Int[] + +function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr) + o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) + o == C.PyNULL && return C.PyNULL + UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL + UnsafePtr{PyJuliaValueObject}(o).value[] = 0 + return o +end + +function _pyjl_dealloc(o::C.PyPtr) + idx = UnsafePtr{PyJuliaValueObject}(o).value[] + if idx != 0 + PYJLVALUES[idx] = nothing + push!(PYJLFREEVALUES, idx) + end + UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o) + ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (C.PyPtr,), o) + nothing +end + +const PYJLMETHODS = Vector{Any}() + +function PyJulia_MethodNum(f) + @nospecialize f + push!(PYJLMETHODS, f) + return length(PYJLMETHODS) +end + +function _pyjl_isnull(o::C.PyPtr, ::C.PyPtr) + ans = PyJuliaValue_IsNull(o) ? C.POINTERS._Py_TrueStruct : C.POINTERS._Py_FalseStruct + C.Py_IncRef(ans) + ans +end + +function _pyjl_callmethod(o::C.PyPtr, args::C.PyPtr) + nargs = C.PyTuple_Size(args) + @assert nargs > 0 + num = C.PyLong_AsLongLong(C.PyTuple_GetItem(args, 0)) + num == -1 && return C.PyNULL + f = PYJLMETHODS[num] + # this form gets defined in jlwrap/base.jl + return _pyjl_callmethod(f, o, args, nargs)::C.PyPtr +end + +const PYJLBUFCACHE = Dict{Ptr{Cvoid},Any}() + +@kwdef struct PyBufferInfo{N} + # data + ptr::Ptr{Cvoid} + readonly::Bool + # items + itemsize::Int + format::String + # layout + shape::NTuple{N,Int} + strides::NTuple{N,Int} + suboffsets::NTuple{N,Int} = ntuple(i -> -1, N) +end + +_pyjl_get_buffer_impl(obj::C.PyPtr, buf::Ptr{C.Py_buffer}, flags::Cint, x, f) = _pyjl_get_buffer_impl(obj, buf, flags, f(x)::PyBufferInfo) + +function _pyjl_get_buffer_impl(obj::C.PyPtr, buf::Ptr{C.Py_buffer}, flags::Cint, info::PyBufferInfo{N}) where {N} + b = UnsafePtr(buf) + c = [] + + # not influenced by flags: obj, buf, len, itemsize, ndim + b.obj[] = C_NULL + b.buf[] = info.ptr + b.itemsize[] = info.itemsize + b.len[] = info.itemsize * prod(info.shape) + b.ndim[] = N + + # readonly + if !info.readonly + b.readonly[] = 0 + elseif Utils.isflagset(flags, C.PyBUF_WRITABLE) + C.PyErr_SetString(C.POINTERS.PyExc_BufferError, "not writable") + return Cint(-1) + else + b.readonly[] = 1 + end + + # format + if Utils.isflagset(flags, C.PyBUF_FORMAT) + push!(c, info.format) + b.format[] = pointer(info.format) + else + b.format[] = C_NULL + end + + # shape + if Utils.isflagset(flags, C.PyBUF_ND) + shape = C.Py_ssize_t[info.shape...] + push!(c, shape) + b.shape[] = pointer(shape) + else + b.shape[] = C_NULL + end + + # strides + if Utils.isflagset(flags, C.PyBUF_STRIDES) + strides = C.Py_ssize_t[info.strides...] + push!(c, strides) + b.strides[] = pointer(strides) + elseif Utils.size_to_cstrides(info.itemsize, info.shape) == info.strides + b.strides[] = C_NULL + else + C.PyErr_SetString(C.POINTERS.PyExc_BufferError, "not C contiguous and strides not requested") + return Cint(-1) + end + + # suboffsets + if all(==(-1), info.suboffsets) + b.suboffsets[] = C_NULL + elseif Utils.isflagset(flags, C.PyBUF_INDIRECT) + suboffsets = C.Py_ssize_t[info.suboffsets...] + push!(c, suboffsets) + b.suboffsets[] = pointer(suboffsets) + else + C.PyErr_SetString(C.POINTERS.PyExc_BufferError, "indirect array and suboffsets not requested") + return Cint(-1) + end + + # check contiguity + if Utils.isflagset(flags, C.PyBUF_C_CONTIGUOUS) + if Utils.size_to_cstrides(info.itemsize, info.shape) != info.strides + C.PyErr_SetString(C.POINTERS.PyExc_BufferError, "not C contiguous") + return Cint(-1) + end + end + if Utils.isflagset(flags, C.PyBUF_F_CONTIGUOUS) + if Utils.size_to_fstrides(info.itemsize, info.shape) != info.strides + C.PyErr_SetString(C.POINTERS.PyExc_BufferError, "not Fortran contiguous") + return Cint(-1) + end + end + if Utils.isflagset(flags, C.PyBUF_ANY_CONTIGUOUS) + if Utils.size_to_cstrides(info.itemsize, info.shape) != info.strides && + Utils.size_to_fstrides(info.itemsize, info.shape) != info.strides + C.PyErr_SetString(C.POINTERS.PyExc_BufferError, "not contiguous") + return Cint(-1) + end + end + + # internal + cptr = Base.pointer_from_objref(c) + PYJLBUFCACHE[cptr] = c + b.internal[] = cptr + + # obj + C.Py_IncRef(obj) + b.obj[] = obj + Cint(0) +end + +function _pyjl_get_buffer(o::C.PyPtr, buf::Ptr{C.Py_buffer}, flags::Cint) + num_ = C.PyObject_GetAttrString(o, "_jl_buffer_info") + num_ == C_NULL && (C.PyErr_Clear(); C.PyErr_SetString(C.POINTERS.PyExc_BufferError, "not a buffer"); return Cint(-1)) + num = C.PyLong_AsLongLong(num_) + C.Py_DecRef(num_) + num == -1 && return Cint(-1) + try + f = PYJLMETHODS[num] + x = PyJuliaValue_GetValue(o) + return _pyjl_get_buffer_impl(o, buf, flags, x, f)::Cint + catch exc + @debug "error getting the buffer" exc + C.PyErr_SetString(C.POINTERS.PyExc_BufferError, "some error occurred getting the buffer") + return Cint(-1) + end +end + +function _pyjl_release_buffer(xo::C.PyPtr, buf::Ptr{C.Py_buffer}) + delete!(PYJLBUFCACHE, UnsafePtr(buf).internal[!]) + nothing +end + +function _pyjl_reduce(self::C.PyPtr, ::C.PyPtr) + v = _pyjl_serialize(self, C.PyNULL) + v == C.PyNULL && return C.PyNULL + args = C.PyTuple_New(1) + args == C.PyNULL && (C.Py_DecRef(v); return C.PyNULL) + err = C.PyTuple_SetItem(args, 0, v) + err == -1 && (C.Py_DecRef(args); return C.PyNULL) + red = C.PyTuple_New(2) + red == C.PyNULL && (C.Py_DecRef(args); return C.PyNULL) + err = C.PyTuple_SetItem(red, 1, args) + err == -1 && (C.Py_DecRef(red); return C.PyNULL) + f = C.PyObject_GetAttrString(self, "_jl_deserialize") + f == C.PyNULL && (C.Py_DecRef(red); return C.PyNULL) + err = C.PyTuple_SetItem(red, 0, f) + err == -1 && (C.Py_DecRef(red); return C.PyNULL) + return red +end + +function _pyjl_serialize(self::C.PyPtr, ::C.PyPtr) + try + io = IOBuffer() + serialize(io, PyJuliaValue_GetValue(self)) + b = take!(io) + return C.PyBytes_FromStringAndSize(pointer(b), sizeof(b)) + catch e + C.PyErr_SetString(C.POINTERS.PyExc_Exception, "error serializing this value") + # wrap sprint in another try-catch block to prevent this function from throwing + try + @debug "Caught exception $(sprint(showerror, e, catch_backtrace()))" + catch + end + return C.PyNULL + end +end + +function _pyjl_deserialize(t::C.PyPtr, v::C.PyPtr) + try + ptr = Ref{Ptr{Cchar}}() + len = Ref{C.Py_ssize_t}() + err = C.PyBytes_AsStringAndSize(v, ptr, len) + err == -1 && return C.PyNULL + io = IOBuffer(unsafe_wrap(Array, Ptr{UInt8}(ptr[]), Int(len[]))) + x = deserialize(io) + return PyJuliaValue_New(t, x) + catch e + C.PyErr_SetString(C.POINTERS.PyExc_Exception, "error deserializing this value") + # wrap sprint in another try-catch block to prevent this function from throwing + try + @debug "Caught exception $(sprint(showerror, e, catch_backtrace()))" + catch + end + return C.PyNULL + end +end + +const _pyjlbase_name = "juliacall.ValueBase" +const _pyjlbase_type = fill(C.PyTypeObject()) +const _pyjlbase_isnull_name = "_jl_isnull" +const _pyjlbase_callmethod_name = "_jl_callmethod" +const _pyjlbase_reduce_name = "__reduce__" +const _pyjlbase_serialize_name = "_jl_serialize" +const _pyjlbase_deserialize_name = "_jl_deserialize" +const _pyjlbase_methods = Vector{C.PyMethodDef}() +const _pyjlbase_as_buffer = fill(C.PyBufferProcs()) + +function init_c() + empty!(_pyjlbase_methods) + push!(_pyjlbase_methods, + C.PyMethodDef( + name = pointer(_pyjlbase_callmethod_name), + meth = @cfunction(_pyjl_callmethod, C.PyPtr, (C.PyPtr, C.PyPtr)), + flags = C.Py_METH_VARARGS, + ), + C.PyMethodDef( + name = pointer(_pyjlbase_isnull_name), + meth = @cfunction(_pyjl_isnull, C.PyPtr, (C.PyPtr, C.PyPtr)), + flags = C.Py_METH_NOARGS, + ), + C.PyMethodDef( + name = pointer(_pyjlbase_reduce_name), + meth = @cfunction(_pyjl_reduce, C.PyPtr, (C.PyPtr, C.PyPtr)), + flags = C.Py_METH_NOARGS, + ), + C.PyMethodDef( + name = pointer(_pyjlbase_serialize_name), + meth = @cfunction(_pyjl_serialize, C.PyPtr, (C.PyPtr, C.PyPtr)), + flags = C.Py_METH_NOARGS, + ), + C.PyMethodDef( + name = pointer(_pyjlbase_deserialize_name), + meth = @cfunction(_pyjl_deserialize, C.PyPtr, (C.PyPtr, C.PyPtr)), + flags = C.Py_METH_O | C.Py_METH_CLASS, + ), + C.PyMethodDef(), + ) + _pyjlbase_as_buffer[] = C.PyBufferProcs( + get = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint)), + release = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer})), + ) + _pyjlbase_type[] = C.PyTypeObject( + name = pointer(_pyjlbase_name), + basicsize = sizeof(PyJuliaValueObject), + # new = C.POINTERS.PyType_GenericNew, + new = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr)), + dealloc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,)), + flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG, + weaklistoffset = fieldoffset(PyJuliaValueObject, 3), + # getattro = C.POINTERS.PyObject_GenericGetAttr, + # setattro = C.POINTERS.PyObject_GenericSetAttr, + methods = pointer(_pyjlbase_methods), + as_buffer = pointer(_pyjlbase_as_buffer), + ) + o = PyJuliaBase_Type[] = C.PyPtr(pointer(_pyjlbase_type)) + if C.PyType_Ready(o) == -1 + C.PyErr_Print() + error("Error initializing 'juliacall.ValueBase'") + end +end + +function __init__() + C.with_gil() do + init_c() + end +end + +PyJuliaValue_IsNull(o::C.PyPtr) = UnsafePtr{PyJuliaValueObject}(o).value[] == 0 + +PyJuliaValue_GetValue(o::C.PyPtr) = PYJLVALUES[UnsafePtr{PyJuliaValueObject}(o).value[]] + +PyJuliaValue_SetValue(o::C.PyPtr, @nospecialize(v)) = begin + idx = UnsafePtr{PyJuliaValueObject}(o).value[] + if idx == 0 + if isempty(PYJLFREEVALUES) + push!(PYJLVALUES, v) + idx = length(PYJLVALUES) + else + idx = pop!(PYJLFREEVALUES) + PYJLVALUES[idx] = v + end + UnsafePtr{PyJuliaValueObject}(o).value[] = idx + else + PYJLVALUES[idx] = v + end + nothing +end + +PyJuliaValue_New(t::C.PyPtr, @nospecialize(v)) = begin + if C.PyType_IsSubtype(t, PyJuliaBase_Type[]) != 1 + C.PyErr_SetString(C.POINTERS.PyExc_TypeError, "Expecting a subtype of 'juliacall.ValueBase'") + return C.PyNULL + end + o = C.PyObject_CallObject(t, C.PyNULL) + o == C.PyNULL && return C.PyNULL + PyJuliaValue_SetValue(o, v) + return o +end + +end diff --git a/src/jlwrap/_.jl b/src/jlwrap/_.jl new file mode 100644 index 00000000..4ad22787 --- /dev/null +++ b/src/jlwrap/_.jl @@ -0,0 +1,59 @@ +""" + module _jlwrap + +Defines the Python object wrappers around Julia objects (`juliacall.AnyValue` etc). +""" +module _jlwrap + +using ..PythonCall: PythonCall +using .._Py +using .._Py: C, Utils, pynew, @autopy, incref, decref, setptr!, getptr, pyjuliacallmodule, pycopy!, errcheck, errset, PyNULL, pyistuple, pyisnull, pyJuliaError, pydel!, pyistype, pytypecheck, pythrow, pytuple_getitem, pyisslice, pystr_asstring, pyosmodule, pyisstr +using .._pyconvert: pyconvert, @pyconvert, PYCONVERT_PRIORITY_WRAP, pyconvert_add_rule, pyconvert_tryconvert, pyconvertarg, pyconvert_result + +using Pkg: Pkg +using Base: @propagate_inbounds, allocatedinline + +import .._Py: Py + +include("C.jl") +include("base.jl") +include("raw.jl") +include("any.jl") +include("iter.jl") +include("type.jl") +include("module.jl") +include("io.jl") +include("number.jl") +include("objectarray.jl") +include("array.jl") +include("vector.jl") +include("dict.jl") +include("set.jl") +include("callback.jl") + +function __init__() + Cjl.C.with_gil() do + init_base() + init_raw() + init_any() + init_iter() + init_type() + init_module() + init_io() + init_number() + init_array() + init_vector() + init_dict() + init_set() + init_callback() + # add packages to juliacall + jl = pyjuliacallmodule + jl.Core = Core + jl.Base = Base + jl.Main = Main + jl.Pkg = Pkg + jl.PythonCall = PythonCall + end +end + +end diff --git a/src/jlwrap/any.jl b/src/jlwrap/any.jl index 5458faa5..df7f77fc 100644 --- a/src/jlwrap/any.jl +++ b/src/jlwrap/any.jl @@ -172,7 +172,7 @@ function pyjlany_mimebundle(self, include::Py, exclude::Py) return ans end -function init_jlwrap_any() +function init_any() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/array.jl b/src/jlwrap/array.jl index bbfbd08e..23b6c27a 100644 --- a/src/jlwrap/array.jl +++ b/src/jlwrap/array.jl @@ -137,14 +137,14 @@ pyjlarray_isbufferabletype(::Type{T}) where {T} = T in ( ) pyjlarray_isbufferabletype(::Type{T}) where {T<:Tuple} = isconcretetype(T) && - PythonCall.allocatedinline(T) && + allocatedinline(T) && all(pyjlarray_isbufferabletype, fieldtypes(T)) pyjlarray_isbufferabletype(::Type{NamedTuple{names,T}}) where {names,T} = pyjlarray_isbufferabletype(T) function pyjlarray_buffer_info(x::AbstractArray{T,N}) where {T,N} if pyjlarray_isbufferabletype(T) - C.PyBufferInfo{N}( + Cjl.PyBufferInfo{N}( ptr = Base.unsafe_convert(Ptr{T}, x), readonly = !Utils.ismutablearray(x), itemsize = sizeof(T), @@ -287,7 +287,7 @@ function pyjlarray_array_interface(x::AbstractArray{T,N}) where {T,N} end pyjl_handle_error_type(::typeof(pyjlarray_array_interface), x, exc) = pybuiltins.AttributeError -function init_jlwrap_array() +function init_array() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/base.jl b/src/jlwrap/base.jl index e5c31320..dc3f4c06 100644 --- a/src/jlwrap/base.jl +++ b/src/jlwrap/base.jl @@ -1,10 +1,10 @@ const pyjlbasetype = pynew() -_pyjl_getvalue(x) = @autopy x C.PyJuliaValue_GetValue(getptr(x_)) +_pyjl_getvalue(x) = @autopy x Cjl.PyJuliaValue_GetValue(getptr(x_)) -_pyjl_setvalue!(x, v) = @autopy x C.PyJuliaValue_SetValue(getptr(x_), v) +_pyjl_setvalue!(x, v) = @autopy x Cjl.PyJuliaValue_SetValue(getptr(x_), v) -pyjl(t, v) = pynew(errcheck(@autopy t C.PyJuliaValue_New(getptr(t_), v))) +pyjl(t, v) = pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(getptr(t_), v))) """ pyisjl(x) @@ -16,7 +16,7 @@ export pyisjl pyjlisnull(x) = @autopy x begin if pyisjl(x_) - C.PyJuliaValue_IsNull(getptr(x_)) + Cjl.PyJuliaValue_IsNull(getptr(x_)) else error("Expecting a 'juliacall.ValueBase', got a '$(pytype(x_).__name__)'") end @@ -36,21 +36,25 @@ pyjlvalue(x) = @autopy x begin end export pyjlvalue -function init_jlwrap_base() - setptr!(pyjlbasetype, incref(C.POINTERS.PyJuliaBase_Type)) +function init_base() + setptr!(pyjlbasetype, incref(Cjl.PyJuliaBase_Type[])) pyjuliacallmodule.ValueBase = pyjlbasetype + + # conversion rule + priority = PYCONVERT_PRIORITY_WRAP + pyconvert_add_rule("juliacall:ValueBase", Any, pyconvert_rule_jlvalue, priority) end pyconvert_rule_jlvalue(::Type{T}, x::Py) where {T} = pyconvert_tryconvert(T, _pyjl_getvalue(x)) -function C._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssize_t) +function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssize_t) @nospecialize f - if C.PyJuliaValue_IsNull(self_) + if Cjl.PyJuliaValue_IsNull(self_) errset(pybuiltins.TypeError, "Julia object is NULL") return C.PyNULL end in_f = false - self = C.PyJuliaValue_GetValue(self_) + self = Cjl.PyJuliaValue_GetValue(self_) try if nargs == 1 in_f = true @@ -123,10 +127,12 @@ end function pyjl_methodnum(f) @nospecialize f - C.PyJulia_MethodNum(f) + Cjl.PyJulia_MethodNum(f) end function pyjl_handle_error_type(f, self, exc) @nospecialize f self exc PyNULL end + +Py(x) = ispy(x) ? throw(MethodError(Py, (x,))) : pyjl(x) diff --git a/src/jlwrap/callback.jl b/src/jlwrap/callback.jl index 910ecf68..338c373f 100644 --- a/src/jlwrap/callback.jl +++ b/src/jlwrap/callback.jl @@ -32,7 +32,7 @@ function pyjlcallback_call(self, args_::Py, kwargs_::Py) end pyjl_handle_error_type(::typeof(pyjlcallback_call), self, exc::MethodError) = exc.f === self ? pybuiltins.TypeError : PyNULL -function init_jlwrap_callback() +function init_callback() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/dict.jl b/src/jlwrap/dict.jl index 6ed73d7b..4d01674a 100644 --- a/src/jlwrap/dict.jl +++ b/src/jlwrap/dict.jl @@ -31,7 +31,7 @@ end const pyjldicttype = pynew() -function init_jlwrap_dict() +function init_dict() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/io.jl b/src/jlwrap/io.jl index 45bb05a4..85785fee 100644 --- a/src/jlwrap/io.jl +++ b/src/jlwrap/io.jl @@ -201,7 +201,7 @@ function pyjltextio_write(io::IO, s_::Py) end pyjl_handle_error_type(::typeof(pyjltextio_write), io, exc) = exc isa MethodError && exc.f === write ? pybuiltins.ValueError : PyNULL -function init_jlwrap_io() +function init_io() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/iter.jl b/src/jlwrap/iter.jl index cfd62063..90d24537 100644 --- a/src/jlwrap/iter.jl +++ b/src/jlwrap/iter.jl @@ -25,7 +25,7 @@ function pyjliter_next(self::Iterator) end end -function init_jlwrap_iter() +function init_iter() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/module.jl b/src/jlwrap/module.jl index b0b0a403..e88a6a4e 100644 --- a/src/jlwrap/module.jl +++ b/src/jlwrap/module.jl @@ -13,7 +13,7 @@ function pyjlmodule_seval(self::Module, expr::Py) Py(Base.eval(self, Meta.parse(strip(pyconvert(String, expr))))) end -function init_jlwrap_module() +function init_module() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/number.jl b/src/jlwrap/number.jl index d0eb7c1e..8d21f1d9 100644 --- a/src/jlwrap/number.jl +++ b/src/jlwrap/number.jl @@ -76,7 +76,7 @@ function pyjlreal_round(self::Real, ndigits_::Py) end pyjl_handle_error_type(::typeof(pyjlreal_round), self, exc::MethodError) = exc.f === round ? pybuiltins.TypeError : PyNULL -function init_jlwrap_number() +function init_number() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/objectarray.jl b/src/jlwrap/objectarray.jl index 2aab4be2..3aa0bb25 100644 --- a/src/jlwrap/objectarray.jl +++ b/src/jlwrap/objectarray.jl @@ -23,7 +23,7 @@ PyObjectArray(undef::UndefInitializer, dims::Vararg{Integer,N}) where {N} = PyOb PyObjectArray{N}(x::AbstractArray{T,N}) where {T,N} = copyto!(PyObjectArray{N}(undef, size(x)), x) PyObjectArray(x::AbstractArray{T,N}) where {T,N} = PyObjectArray{N}(x) -pyobjectarray_finalizer(x::PyObjectArray) = GC.enqueue_all(x.ptrs) +pyobjectarray_finalizer(x::PyObjectArray) = _Py.GC.enqueue_all(x.ptrs) Base.IndexStyle(x::PyObjectArray) = Base.IndexStyle(x.ptrs) diff --git a/src/jlwrap/raw.jl b/src/jlwrap/raw.jl index da02b8e0..2c97dc1b 100644 --- a/src/jlwrap/raw.jl +++ b/src/jlwrap/raw.jl @@ -73,7 +73,7 @@ end pyjlraw_bool(self::Bool) = Py(self) pyjlraw_bool(self) = (errset(pybuiltins.TypeError, "Only Julia 'Bool' can be tested for truthyness"); PyNULL) -function init_jlwrap_raw() +function init_raw() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/set.jl b/src/jlwrap/set.jl index c4ff1b1f..38c9b82d 100644 --- a/src/jlwrap/set.jl +++ b/src/jlwrap/set.jl @@ -73,7 +73,7 @@ function pyjlset_symmetric_difference_update(x::AbstractSet, vs_::Py) Py(nothing) end -function init_jlwrap_set() +function init_set() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/type.jl b/src/jlwrap/type.jl index e081b8f1..01edee7a 100644 --- a/src/jlwrap/type.jl +++ b/src/jlwrap/type.jl @@ -10,7 +10,7 @@ function pyjltype_getitem(self::Type, k_) end end -function init_jlwrap_type() +function init_type() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/jlwrap/vector.jl b/src/jlwrap/vector.jl index 1b7c4180..48b51550 100644 --- a/src/jlwrap/vector.jl +++ b/src/jlwrap/vector.jl @@ -84,7 +84,6 @@ function pyjlvector_remove(x::AbstractVector, v_::Py) errset(pybuiltins.ValueError, "value not in array") return PyNULL end - v = pyconvert_result(r) k = findfirst(==(v), x) if k === nothing errset(pybuiltins.ValueError, "value not in array") @@ -112,7 +111,7 @@ function pyjlvector_count(x::AbstractVector, v_::Py) Py(count(==(v), x)) end -function init_jlwrap_vector() +function init_vector() jl = pyjuliacallmodule pybuiltins.exec(pybuiltins.compile(""" $("\n"^(@__LINE__()-1)) diff --git a/src/pyconvert/_.jl b/src/pyconvert/_.jl new file mode 100644 index 00000000..e818f745 --- /dev/null +++ b/src/pyconvert/_.jl @@ -0,0 +1,29 @@ +""" + module _pyconvert + +Implements `pyconvert`. +""" +module _pyconvert + +using .._Py +using .._Py: C, Utils, @autopy, getptr, incref, pynew, PyNULL, pyisnull, pydel!, pyisint, iserrset_ambig, pyisnone, pyisTrue, pyisFalse, pyfloat_asdouble, pycomplex_ascomplex, pyisstr, pystr_asstring, pyisbytes, pybytes_asvector, pybytes_asUTF8string, pyisfloat, pyisrange, pytuple_getitem, unsafe_pynext, pyistuple, pydatetimetype, pytime_isaware, pydatetime_isaware, _base_pydatetime, _base_datetime, errmatches, errclear, errset, pyiscomplex, pythrow, pybool_asbool +using Dates: Date, Time, DateTime, Millisecond + +import .._Py: pyconvert + +include("pyconvert.jl") +include("rules.jl") +include("ctypes.jl") +include("numpy.jl") +include("pandas.jl") + +function __init__() + C.with_gil() do + init_pyconvert() + init_ctypes() + init_numpy() + init_pandas() + end +end + +end diff --git a/src/concrete/ctypes.jl b/src/pyconvert/ctypes.jl similarity index 100% rename from src/concrete/ctypes.jl rename to src/pyconvert/ctypes.jl diff --git a/src/concrete/numpy.jl b/src/pyconvert/numpy.jl similarity index 100% rename from src/concrete/numpy.jl rename to src/pyconvert/numpy.jl diff --git a/src/concrete/pandas.jl b/src/pyconvert/pandas.jl similarity index 100% rename from src/concrete/pandas.jl rename to src/pyconvert/pandas.jl diff --git a/src/convert.jl b/src/pyconvert/pyconvert.jl similarity index 90% rename from src/convert.jl rename to src/pyconvert/pyconvert.jl index 21b8ac4e..ff775e66 100644 --- a/src/convert.jl +++ b/src/pyconvert/pyconvert.jl @@ -389,15 +389,6 @@ function init_pyconvert() push!(PYCONVERT_EXTRATYPES, pyimport("numbers"=>("Number", "Complex", "Real", "Rational", "Integral"))...) push!(PYCONVERT_EXTRATYPES, pyimport("collections.abc" => ("Iterable", "Sequence", "Set", "Mapping"))...) - priority = PYCONVERT_PRIORITY_WRAP - pyconvert_add_rule("juliacall:ValueBase", Any, pyconvert_rule_jlvalue, priority) - - priority = PYCONVERT_PRIORITY_ARRAY - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) - priority = PYCONVERT_PRIORITY_CANONICAL pyconvert_add_rule("builtins:NoneType", Nothing, pyconvert_rule_none, priority) pyconvert_add_rule("builtins:bool", Bool, pyconvert_rule_bool, priority) @@ -408,14 +399,6 @@ function init_pyconvert() pyconvert_add_rule("builtins:bytes", Base.CodeUnits{UInt8,String}, pyconvert_rule_bytes, priority) pyconvert_add_rule("builtins:range", StepRange{<:Integer,<:Integer}, pyconvert_rule_range, priority) pyconvert_add_rule("numbers:Rational", Rational{<:Integer}, pyconvert_rule_fraction, priority) - pyconvert_add_rule("collections.abc:Iterable", PyIterable, pyconvert_rule_iterable, priority) - pyconvert_add_rule("collections.abc:Sequence", PyList, pyconvert_rule_sequence, priority) - pyconvert_add_rule("collections.abc:Set", PySet, pyconvert_rule_set, priority) - pyconvert_add_rule("collections.abc:Mapping", PyDict, pyconvert_rule_mapping, priority) - pyconvert_add_rule("io:IOBase", PyIO, pyconvert_rule_io, priority) - pyconvert_add_rule("_io:_IOBase", PyIO, pyconvert_rule_io, priority) - pyconvert_add_rule("pandas.core.frame:DataFrame", PyPandasDataFrame, pyconvert_rule_pandasdataframe, priority) - pyconvert_add_rule("pandas.core.arrays.base:ExtensionArray", PyList, pyconvert_rule_sequence, priority) pyconvert_add_rule("builtins:tuple", NamedTuple, pyconvert_rule_iterable, priority) pyconvert_add_rule("builtins:tuple", Tuple, pyconvert_rule_iterable, priority) pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority) @@ -444,14 +427,6 @@ function init_pyconvert() pyconvert_add_rule("collections.abc:Sequence", Tuple, pyconvert_rule_iterable, priority) pyconvert_add_rule("collections.abc:Set", Set, pyconvert_rule_iterable, priority) pyconvert_add_rule("collections.abc:Mapping", Dict, pyconvert_rule_mapping, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", Array, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) - pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) priority = PYCONVERT_PRIORITY_FALLBACK pyconvert_add_rule("builtins:object", Py, pyconvert_rule_object, priority) diff --git a/src/pyconvert/rules.jl b/src/pyconvert/rules.jl new file mode 100644 index 00000000..8d0dc166 --- /dev/null +++ b/src/pyconvert/rules.jl @@ -0,0 +1,397 @@ +### object + +pyconvert_rule_object(::Type{Py}, x::Py) = pyconvert_return(x) + +### Exception + +pyconvert_rule_exception(::Type{R}, x::Py) where {R<:PyException} = pyconvert_return(PyException(x)) + +### None + +pyconvert_rule_none(::Type{Nothing}, x::Py) = pyconvert_return(nothing) +pyconvert_rule_none(::Type{Missing}, x::Py) = pyconvert_return(missing) + +### Bool + +function pyconvert_rule_bool(::Type{T}, x::Py) where {T<:Number} + val = pybool_asbool(x) + if T in (Bool, Int8, Int16, Int32, Int64, Int128, UInt8, UInt16, UInt32, UInt64, UInt128, BigInt) + pyconvert_return(T(val)) + else + pyconvert_tryconvert(T, val) + end +end + +### str + +pyconvert_rule_str(::Type{String}, x::Py) = pyconvert_return(pystr_asstring(x)) +pyconvert_rule_str(::Type{Symbol}, x::Py) = pyconvert_return(Symbol(pystr_asstring(x))) +pyconvert_rule_str(::Type{Char}, x::Py) = begin + s = pystr_asstring(x) + if length(s) == 1 + pyconvert_return(first(s)) + else + pyconvert_unconverted() + end +end + +### bytes + +pyconvert_rule_bytes(::Type{Vector{UInt8}}, x::Py) = pyconvert_return(copy(pybytes_asvector(x))) +pyconvert_rule_bytes(::Type{Base.CodeUnits{UInt8,String}}, x::Py) = pyconvert_return(codeunits(pybytes_asUTF8string(x))) + +### int + +pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = begin + # first try to convert to Clonglong (or Culonglong if unsigned) + v = T <: Unsigned ? C.PyLong_AsUnsignedLongLong(getptr(x)) : C.PyLong_AsLongLong(getptr(x)) + if !iserrset_ambig(v) + # success + return pyconvert_tryconvert(T, v) + elseif errmatches(pybuiltins.OverflowError) + # overflows Clonglong or Culonglong + errclear() + if T in ( + Bool, + Int8, + Int16, + Int32, + Int64, + Int128, + UInt8, + UInt16, + UInt32, + UInt64, + UInt128, + ) && + typemin(typeof(v)) ≤ typemin(T) && + typemax(T) ≤ typemax(typeof(v)) + # definitely overflows S, give up now + return pyconvert_unconverted() + else + # try converting -> int -> str -> BigInt -> T + x_int = pyint(x) + x_str = pystr(String, x_int) + pydel!(x_int) + v = parse(BigInt, x_str) + return pyconvert_tryconvert(T, v) + end + else + # other error + pythrow() + end +end + +### float + +function pyconvert_rule_float(::Type{T}, x::Py) where {T<:Number} + val = pyfloat_asdouble(x) + if T in (Float16, Float32, Float64, BigFloat) + pyconvert_return(T(val)) + else + pyconvert_tryconvert(T, val) + end +end + +# NaN is sometimes used to represent missing data of other types +# so we allow converting it to Nothing or Missing +function pyconvert_rule_float(::Type{Nothing}, x::Py) + val = pyfloat_asdouble(x) + if isnan(val) + pyconvert_return(nothing) + else + pyconvert_unconverted() + end +end + +function pyconvert_rule_float(::Type{Missing}, x::Py) + val = pyfloat_asdouble(x) + if isnan(val) + pyconvert_return(missing) + else + pyconvert_unconverted() + end +end + +### complex + +function pyconvert_rule_complex(::Type{T}, x::Py) where {T<:Number} + val = pycomplex_ascomplex(x) + if T in (Complex{Float64}, Complex{Float32}, Complex{Float16}, Complex{BigFloat}) + pyconvert_return(T(val)) + else + pyconvert_tryconvert(T, val) + end +end + +### range + +function pyconvert_rule_range(::Type{R}, x::Py, ::Type{StepRange{T0,S0}}=Utils._type_lb(R), ::Type{StepRange{T1,S1}}=Utils._type_ub(R)) where {R<:StepRange,T0,S0,T1,S1} + a = @pyconvert(Utils._typeintersect(Integer, T1), x.start) + b = @pyconvert(Utils._typeintersect(Integer, S1), x.step) + c = @pyconvert(Utils._typeintersect(Integer, T1), x.stop) + a′, c′ = promote(a, c - oftype(c, sign(b))) + T2 = Utils._promote_type_bounded(T0, typeof(a′), typeof(c′), T1) + S2 = Utils._promote_type_bounded(S0, typeof(c′), S1) + pyconvert_return(StepRange{T2, S2}(a′, b, c′)) +end + +function pyconvert_rule_range(::Type{R}, x::Py, ::Type{UnitRange{T0}}=Utils._type_lb(R), ::Type{UnitRange{T1}}=Utils._type_ub(R)) where {R<:UnitRange,T0,T1} + b = @pyconvert(Int, x.step) + b == 1 || return pyconvert_unconverted() + a = @pyconvert(Utils._typeintersect(Integer, T1), x.start) + c = @pyconvert(Utils._typeintersect(Integer, T1), x.stop) + a′, c′ = promote(a, c - oftype(c, 1)) + T2 = Utils._promote_type_bounded(T0, typeof(a′), typeof(c′), T1) + pyconvert_return(UnitRange{T2}(a′, c′)) +end + +### fraction + +# works for any collections.abc.Rational +function pyconvert_rule_fraction(::Type{R}, x::Py, ::Type{Rational{T0}}=Utils._type_lb(R), ::Type{Rational{T1}}=Utils._type_ub(R)) where {R<:Rational,T0,T1} + a = @pyconvert(Utils._typeintersect(Integer, T1), x.numerator) + b = @pyconvert(Utils._typeintersect(Integer, T1), x.denominator) + a, b = promote(a, b) + T2 = Utils._promote_type_bounded(T0, typeof(a), typeof(b), T1) + pyconvert_return(Rational{T2}(a, b)) +end + +# works for any collections.abc.Rational +function pyconvert_rule_fraction(::Type{T}, x::Py) where {T<:Number} + pyconvert_tryconvert(T, @pyconvert(Rational{<:Integer}, x)) +end + +### collections + +# Vector + +function _pyconvert_rule_iterable(ans::Vector{T0}, it::Py, ::Type{T1}) where {T0,T1} + @label again + x_ = unsafe_pynext(it) + if pyisnull(x_) + pydel!(it) + return pyconvert_return(ans) + end + x = @pyconvert(T1, x_) + if x isa T0 + push!(ans, x) + @goto again + end + T2 = Utils._promote_type_bounded(T0, typeof(x), T1) + ans2 = Vector{T2}(ans) + push!(ans2, x) + return _pyconvert_rule_iterable(ans2, it, T1) +end + +function pyconvert_rule_iterable(::Type{R}, x::Py, ::Type{Vector{T0}}=Utils._type_lb(R), ::Type{Vector{T1}}=Utils._type_ub(R)) where {R<:Vector,T0,T1} + it = pyiter(x) + ans = Vector{T0}() + return _pyconvert_rule_iterable(ans, it, T1) +end + +# Set + +function _pyconvert_rule_iterable(ans::Set{T0}, it::Py, ::Type{T1}) where {T0,T1} + @label again + x_ = unsafe_pynext(it) + if pyisnull(x_) + pydel!(it) + return pyconvert_return(ans) + end + x = @pyconvert(T1, x_) + if x isa T0 + push!(ans, x) + @goto again + end + T2 = Utils._promote_type_bounded(T0, typeof(x), T1) + ans2 = Set{T2}(ans) + push!(ans2, x) + return _pyconvert_rule_iterable(ans2, it, T1) +end + +function pyconvert_rule_iterable(::Type{R}, x::Py, ::Type{Set{T0}}=Utils._type_lb(R), ::Type{Set{T1}}=Utils._type_ub(R)) where {R<:Set,T0,T1} + it = pyiter(x) + ans = Set{T0}() + return _pyconvert_rule_iterable(ans, it, T1) +end + +# Dict + +function _pyconvert_rule_mapping(ans::Dict{K0,V0}, x::Py, it::Py, ::Type{K1}, ::Type{V1}) where {K0,V0,K1,V1} + @label again + k_ = unsafe_pynext(it) + if pyisnull(k_) + pydel!(it) + return pyconvert_return(ans) + end + v_ = pygetitem(x, k_) + k = @pyconvert(K1, k_) + v = @pyconvert(V1, v_) + if k isa K0 && v isa V0 + push!(ans, k => v) + @goto again + end + K2 = Utils._promote_type_bounded(K0, typeof(k), K1) + V2 = Utils._promote_type_bounded(V0, typeof(v), V1) + ans2 = Dict{K2,V2}(ans) + push!(ans2, k => v) + return _pyconvert_rule_mapping(ans2, x, it, K1, V1) +end + +function pyconvert_rule_mapping(::Type{R}, x::Py, ::Type{Dict{K0,V0}}=Utils._type_lb(R), ::Type{Dict{K1,V1}}=Utils._type_ub(R)) where {R<:Dict,K0,V0,K1,V1} + it = pyiter(x) + ans = Dict{K0,V0}() + return _pyconvert_rule_mapping(ans, x, it, K1, V1) +end + +# Tuple + +function pyconvert_rule_iterable(::Type{T}, xs::Py) where {T<:Tuple} + T isa DataType || return pyconvert_unconverted() + if T != Tuple{} && Tuple{T.parameters[end]} == Base.tuple_type_tail(Tuple{T.parameters[end]}) + isvararg = true + vartype = Base.tuple_type_head(Tuple{T.parameters[end]}) + ts = T.parameters[1:end-1] + else + isvararg = false + vartype = Union{} + ts = T.parameters + end + zs = Any[] + for x in xs + if length(zs) < length(ts) + t = ts[length(zs) + 1] + elseif isvararg + t = vartype + else + return pyconvert_unconverted() + end + z = @pyconvert(t, x) + push!(zs, z) + end + return length(zs) < length(ts) ? pyconvert_unconverted() : pyconvert_return(T(zs)) +end + +for N in 0:16 + Ts = [Symbol("T", n) for n in 1:N] + zs = [Symbol("z", n) for n in 1:N] + # Tuple with N elements + @eval function pyconvert_rule_iterable(::Type{Tuple{$(Ts...)}}, xs::Py) where {$(Ts...)} + xs = pytuple(xs) + n = pylen(xs) + n == $N || return pyconvert_unconverted() + $(( + :($z = @pyconvert($T, pytuple_getitem(xs, $(i-1)))) + for (i, T, z) in zip(1:N, Ts, zs) + )...) + return pyconvert_return(($(zs...),)) + end + # Tuple with N elements plus Vararg + @eval function pyconvert_rule_iterable(::Type{Tuple{$(Ts...),Vararg{V}}}, xs::Py) where {$(Ts...),V} + xs = pytuple(xs) + n = pylen(xs) + n ≥ $N || return pyconvert_unconverted() + $(( + :($z = @pyconvert($T, pytuple_getitem(xs, $(i-1)))) + for (i, T, z) in zip(1:N, Ts, zs) + )...) + vs = V[] + for i in $(N+1):n + v = @pyconvert(V, pytuple_getitem(xs, i-1)) + push!(vs, v) + end + return pyconvert_return(($(zs...), vs...)) + end +end + +# Pair + +function pyconvert_rule_iterable(::Type{R}, x::Py, ::Type{Pair{K0,V0}}=Utils._type_lb(R), ::Type{Pair{K1,V1}}=Utils._type_ub(R)) where {R<:Pair,K0,V0,K1,V1} + it = pyiter(x) + k_ = unsafe_pynext(it) + if pyisnull(k_) + pydel!(it) + pydel!(k_) + return pyconvert_unconverted() + end + k = @pyconvert(K1, k_) + v_ = unsafe_pynext(it) + if pyisnull(v_) + pydel!(it) + pydel!(v_) + return pyconvert_unconverted() + end + v = @pyconvert(V1, v_) + z_ = unsafe_pynext(it) + pydel!(it) + if pyisnull(z_) + pydel!(z_) + else + pydel!(z_) + return pyconvert_unconverted() + end + K2 = Utils._promote_type_bounded(K0, typeof(k), K1) + V2 = Utils._promote_type_bounded(V0, typeof(v), V1) + return pyconvert_return(Pair{K2,V2}(k, v)) +end + +# NamedTuple + +_nt_names_types(::Type) = nothing +_nt_names_types(::Type{NamedTuple}) = (nothing, nothing) +_nt_names_types(::Type{NamedTuple{names}}) where {names} = (names, nothing) +_nt_names_types(::Type{NamedTuple{names,types} where {names}}) where {types} = (nothing, types) +_nt_names_types(::Type{NamedTuple{names,types}}) where {names,types} = (names, types) + +function pyconvert_rule_iterable(::Type{R}, x::Py) where {R<:NamedTuple} + # this is actually strict and only converts python named tuples (i.e. tuples with a + # _fields attribute) where the field names match those from R (if specified). + names_types = _nt_names_types(R) + names_types === nothing && return pyconvert_unconverted() + names, types = names_types + pyistuple(x) || return pyconvert_unconverted() + names2_ = pygetattr(x, "_fields", pybuiltins.None) + names2 = @pyconvert(names === nothing ? Tuple{Vararg{Symbol}} : typeof(names), names2_) + pydel!(names2_) + names === nothing || names === names2 || return pyconvert_unconverted() + types2 = types === nothing ? NTuple{length(names2),Any} : types + vals = @pyconvert(types2, x) + length(vals) == length(names2) || return pyconvert_unconverted() + types3 = types === nothing ? typeof(vals) : types + return pyconvert_return(NamedTuple{names2,types3}(vals)) +end + +### datetime + + +function pyconvert_rule_date(::Type{Date}, x::Py) + # datetime is a subtype of date, but we shouldn't convert datetime to Date since it's lossy + pyisinstance(x, pydatetimetype) && return pyconvert_unconverted() + year = pyconvert(Int, x.year) + month = pyconvert(Int, x.month) + day = pyconvert(Int, x.day) + pyconvert_return(Date(year, month, day)) +end + +function pyconvert_rule_time(::Type{Time}, x::Py) + pytime_isaware(x) && return pyconvert_unconverted() + hour = pyconvert(Int, x.hour) + minute = pyconvert(Int, x.minute) + second = pyconvert(Int, x.second) + microsecond = pyconvert(Int, x.microsecond) + return pyconvert_return(Time(hour, minute, second, div(microsecond, 1000), mod(microsecond, 1000))) +end + +function pyconvert_rule_datetime(::Type{DateTime}, x::Py) + pydatetime_isaware(x) && return pyconvert_unconverted() + # compute the time since _base_datetime + # this accounts for fold + d = x - _base_pydatetime + days = pyconvert(Int, d.days) + seconds = pyconvert(Int, d.seconds) + microseconds = pyconvert(Int, d.microseconds) + pydel!(d) + iszero(mod(microseconds, 1000)) || return pyconvert_unconverted() + return pyconvert_return(_base_datetime + Millisecond(div(microseconds, 1000) + 1000 * (seconds + 60 * 60 * 24 * days))) +end diff --git a/src/py_macro.jl b/src/pymacro/_.jl similarity index 98% rename from src/py_macro.jl rename to src/pymacro/_.jl index 98ac0175..de8a32b3 100644 --- a/src/py_macro.jl +++ b/src/pymacro/_.jl @@ -11,6 +11,18 @@ # - generator functions?? # - splatting +""" + module _pymacro + +Provides the `@py` macro. +""" +module _pymacro + +using .._Py +using .._Py: pyisnot, pynotin, BUILTINS, pynew, pycallargs, pydel!, pycopy!, pystr_intern!, pynulltuple, pytuple_setitem, pyset_add, pyisnull, unsafe_pynext, pydict_setitem, pylist_setitem, pynulllist, pybool_asbool, pythrow + +using MacroTools: MacroTools, @capture, isexpr + const PY_MACRO_NILOPS = Dict( :help => (pyhelp, false), :int => (pyint, true), @@ -795,3 +807,5 @@ macro py(ex) esc(py_macro(ex, __module__, __source__)) end export @py + +end diff --git a/src/pywrap/PyIO.jl b/src/pywrap/PyIO.jl index 129600ea..e23f5caa 100644 --- a/src/pywrap/PyIO.jl +++ b/src/pywrap/PyIO.jl @@ -76,7 +76,7 @@ function PyIO(f::Function, o; opts...) try return f(io) finally - pydel!(io) + pydel!(io.py) end end diff --git a/src/pywrap/PyPandasDataFrame.jl b/src/pywrap/PyPandasDataFrame.jl index 5d512365..0c6ab85a 100644 --- a/src/pywrap/PyPandasDataFrame.jl +++ b/src/pywrap/PyPandasDataFrame.jl @@ -28,18 +28,6 @@ pyconvert_rule_pandasdataframe(::Type{PyPandasDataFrame}, x::Py) = pyconvert_ret ### Show -function Base.show(io::IO, mime::MIME"text/plain", df::PyPandasDataFrame) - nrows = pyconvert(Int, @py df.shape[0]) - ncols = pyconvert(Int, @py df.shape[1]) - printstyled(io, nrows, '×', ncols, ' ', typeof(df), '\n', bold=true) - pyshow(io, mime, df) -end - -Base.show(io::IO, mime::MIME, df::PyPandasDataFrame) = pyshow(io, mime, df) -Base.show(io::IO, mime::MIME"text/csv", df::PyPandasDataFrame) = pyshow(io, mime, df) -Base.show(io::IO, mime::MIME"text/tab-separated-values", df::PyPandasDataFrame) = pyshow(io, mime, df) -Base.showable(mime::MIME, df::PyPandasDataFrame) = pyshowable(mime, df) - ### Tables Tables.istable(::Type{PyPandasDataFrame}) = true diff --git a/src/pywrap/_.jl b/src/pywrap/_.jl new file mode 100644 index 00000000..bb50b9fa --- /dev/null +++ b/src/pywrap/_.jl @@ -0,0 +1,57 @@ +""" + module _pywrap + +Defines Julia wrappers around Python objects, including `PyList`, `PyDict`, `PyArray` and `PyIO`. +""" +module _pywrap + +using .._Py +using .._Py: C, Utils, @autopy, unsafe_pynext, pyisnull, PyNULL, getptr, pydel!, pybytes_asvector, pystr_asUTF8vector, pystr_fromUTF8, incref, decref, pynew, pyisnone, pyistuple, pyisstr +using .._pyconvert: pyconvert, pyconvert_tryconvert, pyconvert_unconverted, pyconvert_isunconverted, pyconvert_return, pyconvert_result +using .._pymacro + +using Base: @propagate_inbounds +using Tables: Tables +using UnsafePointers: UnsafePtr + +import .._Py: Py, ispy +import .._pyconvert: pyconvert_add_rule, PYCONVERT_PRIORITY_ARRAY, PYCONVERT_PRIORITY_CANONICAL, PYCONVERT_PRIORITY_NORMAL + +include("PyIterable.jl") +include("PyDict.jl") +include("PyList.jl") +include("PySet.jl") +include("PyArray.jl") +include("PyIO.jl") +include("PyTable.jl") +include("PyPandasDataFrame.jl") + +function __init__() + priority = PYCONVERT_PRIORITY_ARRAY + pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) + pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) + pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) + pyconvert_add_rule("", PyArray, pyconvert_rule_array_nocopy, priority) + + priority = PYCONVERT_PRIORITY_CANONICAL + pyconvert_add_rule("collections.abc:Iterable", PyIterable, pyconvert_rule_iterable, priority) + pyconvert_add_rule("collections.abc:Sequence", PyList, pyconvert_rule_sequence, priority) + pyconvert_add_rule("collections.abc:Set", PySet, pyconvert_rule_set, priority) + pyconvert_add_rule("collections.abc:Mapping", PyDict, pyconvert_rule_mapping, priority) + pyconvert_add_rule("io:IOBase", PyIO, pyconvert_rule_io, priority) + pyconvert_add_rule("_io:_IOBase", PyIO, pyconvert_rule_io, priority) + pyconvert_add_rule("pandas.core.frame:DataFrame", PyPandasDataFrame, pyconvert_rule_pandasdataframe, priority) + pyconvert_add_rule("pandas.core.arrays.base:ExtensionArray", PyList, pyconvert_rule_sequence, priority) + + priority = PYCONVERT_PRIORITY_NORMAL + pyconvert_add_rule("", Array, pyconvert_rule_array, priority) + pyconvert_add_rule("", Array, pyconvert_rule_array, priority) + pyconvert_add_rule("", Array, pyconvert_rule_array, priority) + pyconvert_add_rule("", Array, pyconvert_rule_array, priority) + pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) + pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) + pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) + pyconvert_add_rule("", AbstractArray, pyconvert_rule_array, priority) +end + +end diff --git a/src/pywrap/convert_rules.jl b/src/pywrap/convert_rules.jl new file mode 100644 index 00000000..e69de29b diff --git a/src/utils.jl b/src/utils/_.jl similarity index 99% rename from src/utils.jl rename to src/utils/_.jl index 921ef6d6..f258c196 100644 --- a/src/utils.jl +++ b/src/utils/_.jl @@ -1,4 +1,4 @@ -module Utils +module _Utils function explode_union(T) @nospecialize T diff --git a/srcold/PythonCall.jl b/srcold/PythonCall.jl new file mode 100644 index 00000000..b76b215c --- /dev/null +++ b/srcold/PythonCall.jl @@ -0,0 +1,140 @@ +module PythonCall + +const VERSION = v"0.9.15" +const ROOT_DIR = dirname(@__DIR__) + +using Base: @propagate_inbounds +using MacroTools, Dates, Tables, Markdown, Serialization, Requires, Pkg, REPL + +include("utils.jl") + +include("cpython/CPython.jl") + +include("gc.jl") +include("Py.jl") +include("err.jl") +include("config.jl") +include("convert.jl") +# abstract interfaces +include("abstract/object.jl") +include("abstract/iter.jl") +include("abstract/builtins.jl") +include("abstract/number.jl") +include("abstract/collection.jl") +# concrete types +include("concrete/import.jl") +include("concrete/consts.jl") +include("concrete/str.jl") +include("concrete/bytes.jl") +include("concrete/tuple.jl") +include("concrete/list.jl") +include("concrete/dict.jl") +include("concrete/bool.jl") +include("concrete/int.jl") +include("concrete/float.jl") +include("concrete/complex.jl") +include("concrete/set.jl") +include("concrete/slice.jl") +include("concrete/range.jl") +include("concrete/none.jl") +include("concrete/type.jl") +include("concrete/fraction.jl") +include("concrete/datetime.jl") +include("concrete/code.jl") +include("concrete/ctypes.jl") +include("concrete/numpy.jl") +include("concrete/pandas.jl") +# @py +# anything below can depend on @py, anything above cannot +include("py_macro.jl") +# jlwrap +include("jlwrap/base.jl") +include("jlwrap/raw.jl") +include("jlwrap/callback.jl") +include("jlwrap/any.jl") +include("jlwrap/module.jl") +include("jlwrap/type.jl") +include("jlwrap/iter.jl") +include("jlwrap/objectarray.jl") +include("jlwrap/array.jl") +include("jlwrap/vector.jl") +include("jlwrap/dict.jl") +include("jlwrap/set.jl") +include("jlwrap/number.jl") +include("jlwrap/io.jl") +# pywrap +include("pywrap/PyIterable.jl") +include("pywrap/PyList.jl") +include("pywrap/PySet.jl") +include("pywrap/PyDict.jl") +include("pywrap/PyArray.jl") +include("pywrap/PyIO.jl") +include("pywrap/PyTable.jl") +include("pywrap/PyPandasDataFrame.jl") +# misc +include("pyconst_macro.jl") +include("juliacall.jl") +include("compat/stdlib.jl") +include("compat/with.jl") +include("compat/multimedia.jl") +include("compat/serialization.jl") +include("compat/gui.jl") +include("compat/ipython.jl") +include("compat/tables.jl") + +function __init__() + C.with_gil() do + init_consts() + init_pyconvert() + init_datetime() + # juliacall/jlwrap + init_juliacall() + init_jlwrap_base() + init_jlwrap_raw() + init_jlwrap_callback() + init_jlwrap_any() + init_jlwrap_module() + init_jlwrap_type() + init_jlwrap_iter() + init_jlwrap_array() + init_jlwrap_vector() + init_jlwrap_dict() + init_jlwrap_set() + init_jlwrap_number() + init_jlwrap_io() + init_juliacall_2() + # compat + init_stdlib() + init_pyshow() + init_gui() + init_tables() + init_ctypes() + init_numpy() + init_pandas() + end + @require PyCall="438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall) +end + +function init_pycall(PyCall::Module) + # allow explicit conversion between PythonCall.Py and PyCall.PyObject + # provided they are using the same interpretr + errmsg = """ + Conversion between `PyCall.PyObject` and `PythonCall.Py` is only possible when using the same Python interpreter. + + There are two ways to achieve this: + - Set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`. This forces PythonCall to use the same + interpreter as PyCall, but PythonCall loses the ability to manage its own dependencies. + - Set the environment variable `PYTHON` to `PythonCall.C.CTX.exe_path` and rebuild PyCall. This forces PyCall + to use the same interpreter as PythonCall, but needs to be repeated whenever you switch Julia environment. + """ + @eval function Py(x::$PyCall.PyObject) + C.CTX.matches_pycall::Bool || error($errmsg) + return pynew(C.PyPtr($PyCall.pyreturn(x))) + end + @eval function $PyCall.PyObject(x::Py) + C.CTX.matches_pycall::Bool || error($errmsg) + return $PyCall.PyObject($PyCall.PyPtr(incref(getptr(x)))) + end +end + +end diff --git a/test/jlwrap.jl b/test/jlwrap.jl index 1310054b..1e7e24cd 100644 --- a/test/jlwrap.jl +++ b/test/jlwrap.jl @@ -1,5 +1,5 @@ @testitem "any" begin - struct Foo + mutable struct Foo value::Int end Base.:(+)(x::Foo) = "+ $(x.value)" @@ -17,6 +17,13 @@ Base.:(>>)(x::Foo, y::Foo) = "$(x.value) >> $(y.value)" Base.:(&)(x::Foo, y::Foo) = "$(x.value) & $(y.value)" Base.:(|)(x::Foo, y::Foo) = "$(x.value) | $(y.value)" + Base.powermod(x::Foo, y::Foo, z::Foo) = "powermod($(x.value), $(y.value), $(z.value))" + (x::Foo)(args...; kw...) = "$(x.value)($args)$(length(kw))" + Base.getindex(x::Foo, idx...) = "$(x.value)[$idx]" + Base.setindex!(x::Foo, v, idx...) = (x.value = v+sum(idx); x) + Base.delete!(x::Foo, idx...) = (x.value = -sum(idx); x) + Base.in(v::Int, x::Foo) = x.value == v + Base.nameof(x::Foo) = "nameof $(x.value)" @testset "type" begin @test pyis(pytype(pyjl(Foo(1))), PythonCall.pyjlanytype) @test pyis(pytype(pyjl(nothing)), PythonCall.pyjlanytype) @@ -28,20 +35,69 @@ @test pytruth(pyjl(nothing)) @test pytruth(pyjl(missing)) end + @testset "repr" begin + @test pyrepr(String, pyjl(missing)) == "Julia: missing" + end + @testset "str" begin + @test pystr(String, pyjl(missing)) == "missing" + end + @testset "getattr" begin + @test pyconvert(Int, pygetattr(pyjl(Foo(12)), "value")) === 12 + end + @testset "setattr" begin + x = Foo(12) + @test x.value == 12 + pysetattr(x, "value", 34) + @test x.value == 34 + end + @testset "dir" begin + @test pycontains(pydir(pyjl(Foo(99))), "value") + end + @testset "call" begin + z = pyjl(Foo(1))(4, 5) + @test pyconvert(String, z) == "1((4, 5))0" + z = pyjl(Foo(1))(4, 5; foo=true, bar=true) + @test pyconvert(String, z) == "1((4, 5))2" + end + @testset "getitem" begin + z = pygetitem(pyjl(Foo(1)), 3) + @test pyconvert(String, z) == "1[(3,)]" + z = pygetitem(pyjl(Foo(1)), (4, 5)) + @test pyconvert(String, z) == "1[(4, 5)]" + end + @testset "setitem" begin + z = Foo(0) + x = pyjl(z) + pysetitem(x, 10, 3) + @test z.value == 13 + pysetitem(x, (10, 10), 4) + @test z.value == 24 + end + @testset "delitem" begin + z = Foo(0) + x = pyjl(z) + pydelitem(x, 9) + @test z.value == -9 + pydelitem(x, (3, 4)) + @test z.value == -7 + end + @testset "contains" begin + @test pycontains(pyjl(Foo(45)), 45) + end @testset "pos" begin - z = pyjl(+Foo(1)) + z = +(pyjl(Foo(1))) @test pyconvert(String, z) == "+ 1" end @testset "neg" begin - z = pyjl(-Foo(1)) + z = -(pyjl(Foo(1))) @test pyconvert(String, z) == "- 1" end @testset "abs" begin - z = pyjl(abs(Foo(1))) + z = abs(pyjl(Foo(1))) @test pyconvert(String, z) == "abs 1" end @testset "inv" begin - z = pyjl(~Foo(1)) + z = ~(pyjl(Foo(1))) @test pyconvert(String, z) == "~ 1" end @testset "add" begin @@ -132,6 +188,31 @@ z = pyjlraw(Foo(1)) | pyjl(Foo(2)) @test pyconvert(String, z) == "1 | 2" end + @testset "pow3" begin + z = pypow(pyjl(Foo(1)), pyjl(Foo(2)), pyjl(Foo(3))) + @test pyconvert(String, z) == "powermod(1, 2, 3)" + end + @testset "rpow3" begin + z = pyjl(Foo(2)).__rpow__(pyjl(Foo(1)), pyjl(Foo(3))) + @test pyconvert(String, z) == "powermod(1, 2, 3)" + end + @testset "name" begin + z = pyjl(Foo(135)).__name__ + @test pyconvert(String, z) == "nameof 135" + end + @testset "mimebundle" begin + z = pyjl(Foo(1))._repr_mimebundle_() + @test pyisinstance(z, pybuiltins.dict) + @test pycontains(z, "text/plain") + end + @testset "display" begin + pyjl(Foo(1))._jl_display() + pyjl(Foo(1))._jl_display(mime="text/plain") + end + @testset "help" begin + pyjl(Foo(1))._jl_help() + pyjl(Foo(1))._jl_help(mime="text/plain") + end end @testitem "array" begin diff --git a/test/pywrap.jl b/test/pywrap.jl index 13b4d3a1..6cb9b726 100644 --- a/test/pywrap.jl +++ b/test/pywrap.jl @@ -1,5 +1,5 @@ @testitem "PyArray" begin - x = pyimport("array").array("i", [1,2,3]) + x = pyimport("array").array("i", pylist([1,2,3])) y = PyArray(x) z = PyArray{Cint,1,false,false,Cint}(x) @testset "construct" begin @@ -99,6 +99,9 @@ end @testset "iterate" begin @test collect(z) == ["foo" => 12] end + @testset "iterate keys" begin + @test collect(keys(z)) == ["foo"] + end @testset "getindex" begin @test z["foo"] === 12 @test_throws KeyError z["bar"] @@ -339,6 +342,25 @@ end end @testitem "PyPandasDataFrame" begin + using Tables + @test PyPandasDataFrame isa Type + # TODO: figure out how to get pandas into the test environment + # for now use some dummy type and take advantage of the fact that the code doesn't actually check it's a real dataframe + @pyexec """ + class DataFrame: + def __init__(self, **kw): + self.__dict__.update(kw) + """ => DataFrame + df = DataFrame(shape=(4, 3), columns=pylist(["foo", "bar", "baz"])) + x = PyPandasDataFrame(df) + @test ispy(x) + @test Py(x) === df + @test Tables.istable(x) + @test Tables.columnaccess(x) + @test_throws Exception Tables.columns(x) + @test_throws Exception pyconvert(PyPandasDataFrame, 1) + str = sprint(show, MIME("text/plain"), x) + @test occursin(r"4×3 .*PyPandasDataFrame", str) end @testitem "PySet" begin @@ -441,4 +463,7 @@ end end @testitem "PyTable" begin + # TODO: figure out how to get pandas into the test environment + @test PyTable isa Type + @test_throws r"cannot convert this Python 'int' to a Julia '.*PyTable'" PyTable(0) end diff --git a/test/utils.jl b/test/utils.jl index a0d38492..2f195fd9 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,11 +1,11 @@ @testitem "mimes_for" begin for x in Any[1, "foo", [], 'z'] - @test PythonCall.Utils.mimes_for(x) isa Vector{String} + @test PythonCall._Utils.mimes_for(x) isa Vector{String} end end @testitem "StaticString length and indexing" begin - s = PythonCall.Utils.StaticString{UInt32, 44}("ababababb") + s = PythonCall._Utils.StaticString{UInt32, 44}("ababababb") @test length(s) == 9 @test s[1] == 'a' @test s[1:2] == "ab"