Convert AbstractArrays with strides to NumPy arrays

.github/workflows/aot.yml

@@ -41,13 +41,7 @@ jobs:
           arch: ${{ matrix.architecture }}
           show-versioninfo: true
 
-      # Revert to `@v1` after this PR is merged:
-      # https://github.com/JuliaLang/PackageCompiler.jl/pull/443
-      - run: julia -e 'using Pkg; pkg"add PackageCompiler#cb994c72e2087c57ffa4727ef93589e1b98d8a32"'
-
-      # Workaround https://github.com/JuliaLang/julia/issues/37441.
-      # Once it's solved, we can remove the following line:
-      - run: julia -e 'using Pkg; pkg"dev PyCall"'
+      - run: julia -e 'using Pkg; pkg"add PackageCompiler@v1"'
 
       - run: aot/compile.jl
       - run: aot/assert_has_pycall.jl

aot/runtests.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 thisdir="$(dirname "${BASH_SOURCE[0]}")"
-exec "$thisdir/julia.sh" --startup-file=no --color=yes -e '
+exec "$thisdir/julia.sh" --startup-file=no --color=yes --project=$thisdir -e '
 using Pkg
 Pkg.test("PyCall")
 '

src/PyCall.jl

@@ -8,7 +8,7 @@ using VersionParsing
 
 export pycall, pycall!, pyimport, pyimport_e, pybuiltin, PyObject, PyReverseDims,
        PyPtr, pyincref, pydecref, pyversion,
-       PyArray, PyArray_Info, PyBuffer,
+       PyArray, PyArray_Info, PyBuffer, PySlice,
        pyerr_check, pyerr_clear, pytype_query, PyAny, @pyimport, PyDict,
        pyisinstance, pywrap, pytypeof, pyeval, PyVector, pystring, pystr, pyrepr,
        pyraise, pytype_mapping, pygui, pygui_start, pygui_stop,
@@ -22,7 +22,8 @@ import Base: size, ndims, similar, copy, getindex, setindex!, stride,
        filter!, hash, splice!, pop!, ==, isequal, push!,
        append!, insert!, prepend!, unsafe_convert,
        pushfirst!, popfirst!, firstindex, lastindex,
-       getproperty, setproperty!, propertynames
+       getproperty, setproperty!, propertynames,
+       first, last, step, StridedSubArray, ReinterpretArray, ReshapedArray
 
 if isdefined(Base, :hasproperty) # Julia 1.2
     import Base: hasproperty

src/conversions.jl

@@ -170,7 +170,10 @@ function pyany_toany(T::Type)
 end
 pyany_toany(::Type{PyAny}) = Any
 pyany_toany(t::Type{T}) where {T<:Tuple} = Tuple{map(pyany_toany, t.types)...}
-
+@static if isdefined(Core, :TypeofVararg) # VERSION >= v"1.7.0-DEV.77"
+    # Core.TypeofVararg introduced in https://github.com/JuliaLang/julia/pull/38136
+    pyany_toany(T::Core.TypeofVararg) = T === Vararg{PyAny} ? Vararg{Any} : T
+end
 # PyAny acts like Any for conversions, except for converting PyObject (below)
 convert(::Type{PyAny}, x) = x
 
@@ -230,6 +233,38 @@ function convert(::Type{Pair{K,V}}, o::PyObject) where {K,V}
     return Pair(k, v)
 end
 
+#########################################################################
+# PySlice: no-copy wrapping of a Julia object around a Python slice
+
+slice(start, stop, step) = pycall(pyslice[], PyObject,
+                                  start, stop, step)
+
+struct PySlice{T} <: AbstractRange{T}
+    o::PyObject
+    function PySlice{T}(o::PyObject) where T
+        if !pyisinstance(o, pyslice[])
+            throw(ArgumentError("Argument must be a slice"))
+        end
+        new{T}(o)
+    end
+end
+
+PySlice(stop) = PySlice{typeof(stop)}( slice(nothing, stop, nothing) )
+PySlice(start, stop) = PySlice{promote_type(typeof(start),typeof(stop))}( slice(start, stop, nothing) )
+PySlice(start, stop, step) = PySlice{promote_type(typeof(start),typeof(stop),typeof(step))}( slice(start, stop, step) )
+PySlice(r::AbstractRange{T}) where T = PySlice{T}( slice(first(r), last(r)+1, step(r)) )
+PySlice(S::PySlice{T}) where T = S
+
+first(s::PySlice{T}) where T = (pystart = s.o.start; pystart === nothing ? 0 : pystart)
+last(s::PySlice{T}) where T = s.o.stop-1
+step(s::PySlice{T}) where T = (pystep = s.o.step; pystep === nothing ? 1 : pystep)
+length(s::PySlice{T}) where T = length( first(s):step(s):last(s) )
+
+convert(::Type{PyObject}, S::PySlice) = S.o
+convert(::Type{PySlice}, o::PyObject) = PySlice{Int}(o)
+PyObject(S::PySlice) = S.o
+
+
 #########################################################################
 # PyVector: no-copy wrapping of a Julia object around a Python sequence
 
@@ -612,6 +647,9 @@ function PyObject(r::AbstractRange{T}) where T<:Integer
 end
 
 function convert(::Type{T}, o::PyObject) where T<:AbstractRange
+    if pyisinstance(o, pyslice[])
+        return o.start:o.step:o.stop-1
+    end
     v = PyVector(o)
     len = length(v)
     if len == 0
@@ -751,6 +789,8 @@ function pysequence_query(o::PyObject)
         return typetuple(pytype_query(PyObject(ccall((@pysym :PySequence_GetItem), PyPtr, (PyPtr,Int), o,i-1)), PyAny) for i = 1:len)
     elseif pyisinstance(o, pyxrange[])
         return AbstractRange
+    elseif pyisinstance(o, pyslice[])
+        return PySlice
     elseif ispybytearray(o)
         return Vector{UInt8}
     elseif !isbuftype(o)

src/gc.jl

@@ -29,7 +29,13 @@ const weakref_callback_meth = Ref{PyMethodDef}()
 function pyembed(po::PyObject, jo::Any)
     # If there's a need to support immutable embedding,
     # the API needs to be changed to return the pointer.
-    isimmutable(jo) && throw(ArgumentError("pyembed: immutable argument not allowed"))
+    if isimmutable(jo)
+        if hasmethod(parent, ( typeof(jo), ) )
+            return pyembed(po, parent(jo) )
+        else
+            throw(ArgumentError("pyembed: immutable argument not allowed"))
+        end
+    end
     if ispynull(weakref_callback_obj)
         cf = @cfunction(weakref_callback, PyPtr, (PyPtr,PyPtr))
         weakref_callback_meth[] = PyMethodDef("weakref_callback", cf, METH_O)
@@ -43,3 +49,10 @@ function pyembed(po::PyObject, jo::Any)
     pycall_gc[wo] = jo
     return po
 end
+
+# Embed the mutable type underlying the immutable view of the array
+# See Base.unsafe_convert(::Type{Ptr{T}}, jo::ArrayType) for specific array types
+pyembed(po::PyObject, jo::SubArray         ) = pyembed(po, jo.parent)
+pyembed(po::PyObject, jo::ReshapedArray    ) = pyembed(po, jo.parent)
+pyembed(po::PyObject, jo::ReinterpretArray ) = pyembed(po, jo.parent)
+pyembed(po::PyObject, jo::PermutedDimsArray) = pyembed(po, jo.parent)

src/numpy.jl

@@ -172,7 +172,7 @@ const NPY_ARRAY_WRITEABLE = Int32(0x0400)
 # dimensions. For example, although NumPy works with both row-major and
 # column-major data, some Python libraries like OpenCV seem to require
 # row-major data (the default in NumPy). In such cases, use PyReverseDims(array)
-function NpyArray(a::StridedArray{T}, revdims::Bool) where T<:PYARR_TYPES
+function NpyArray(a::AbstractArray{T}, revdims::Bool) where T<:PYARR_TYPES
     @npyinitialize
     size_a = revdims ? reverse(size(a)) : size(a)
     strides_a = revdims ? reverse(strides(a)) : strides(a)
@@ -186,15 +186,15 @@ function NpyArray(a::StridedArray{T}, revdims::Bool) where T<:PYARR_TYPES
     return PyObject(p, a)
 end
 
-function PyObject(a::StridedArray{T}) where T<:PYARR_TYPES
+function PyObject(a::AbstractArray{T}) where T<:PYARR_TYPES
     try
         return NpyArray(a, false)
     catch
         return array2py(a) # fallback to non-NumPy version
     end
 end
 
-function PyReverseDims(a::StridedArray{T,N}) where {T<:PYARR_TYPES,N}
+function PyReverseDims(a::AbstractArray{T,N}) where {T<:PYARR_TYPES,N}
     try
         return NpyArray(a, true)
     catch
@@ -225,3 +225,23 @@ PyObject(a::Union{LinearAlgebra.Adjoint{<:Real},LinearAlgebra.Transpose}) =
     PyReverseDims(a.parent)
 
 PyObject(a::LinearAlgebra.Adjoint) = PyObject(Matrix(a)) # non-real arrays require a copy
+
+# Alternative conversion of array views mapping parent in Julia to base in Numpy
+PyObjectWithParent(a::AbstractArray) = PyObject(a)
+
+function PyObjectWithParent(pda::PermutedDimsArray{T,N,perm}) where {T,N,perm}
+    parent = PyObjectWithParent(pda.parent)
+    # numpy.transpose is similar to PermutedDimsArray and creates a view of the data
+    pycall(parent.transpose, PyObject, perm .- 1)
+end
+
+function PyObjectWithParent(a::StridedSubArray{T,N}) where {T <: PYARR_TYPES,N}
+    parent = PyObjectWithParent(a.parent)
+    inds = a.indices
+    # hasstep(T) = Val( hasmethod( step, Tuple{ typeof(T) } ) )
+    ind2slice(ind) = isa(ind,AbstractRange) ? 
+                        PySlice(ind .- 1) :
+                        PySlice(ind-1:ind-1)
+    slices = map( ind2slice , inds )
+    pycall( parent.__getitem__, PyObject, slices )
+end

src/pyinit.jl

@@ -20,6 +20,7 @@ const c_void_p_Type = PyNULL()
 # or are simply left as non-const values
 const pynothing = Ref{PyPtr}(0)
 const pyxrange = Ref{PyPtr}(0)
+const pyslice = Ref{PyPtr}(0)
 
 #########################################################################
 # initialize jlWrapType for pytype.jl
@@ -217,6 +218,9 @@ function __init__()
     # xrange type (or range in Python 3)
     pyxrange[] = @pyglobalobj(:PyRange_Type)
 
+    # slice type
+    pyslice[] = @pyglobalobj(:PySlice_Type)
+
     # ctypes.c_void_p for Ptr types
     copy!(c_void_p_Type, pyimport("ctypes")."c_void_p")
 

test/runtests.jl

@@ -72,6 +72,8 @@ const PyInt = pyversion < v"3" ? Int : Clonglong
     @test roundtripeq(Int32)
     @test roundtripeq(Dict(1 => "hello", 2 => "goodbye")) && roundtripeq(Dict())
     @test roundtripeq(UInt8[1,3,4,5])
+    @test roundtripeq(1:5)
+    @test roundtripeq(5:2:10)
     @test roundtrip(3 => 4) == (3,4)
     @test roundtrip(Pair{Int,Int}, 3 => 4) == Pair(3,4)
     @test eltype(roundtrip([Ref(1), Ref(2)])) == typeof(Ref(1))
@@ -87,6 +89,48 @@ const PyInt = pyversion < v"3" ? Int : Clonglong
     @test roundtrip(testkw)(314157) == 314157
     @test roundtrip(testkw)(314157, y=1) == 314159
 
+    let s = PyCall.slice(nothing, 10, nothing)
+        @test s.start === nothing
+        @test s.stop == 10
+        @test s.step === nothing
+        @test PySlice{Int}(s) == 0:9
+        @test convert(PySlice, s) == 0:9
+        @test PyObject( PySlice{Int}(s) ) === s
+    end
+
+    let s = PyCall.slice(1, 5, nothing)
+        @test s.start == 1
+        @test s.stop == 5
+        @test s.step === nothing
+        @test PySlice{Int}(s) == 1:4
+        @test convert(PySlice, s) == 1:4
+        @test PyObject( PySlice{Int}(s) ) === s
+    end
+
+    let s = PyCall.slice(3, 9, 2)
+        @test s.start == 3
+        @test s.stop == 9
+        @test s.step == 2
+        @test PySlice{Int}(s) == 3:2:7
+        @test convert(PySlice, s) == 3:2:7
+        @test PyObject( PySlice{Int}(s) ) === s
+    end
+
+    let s = PySlice(4, 5)
+        @test first(s) == 4
+        @test last(s) == 4
+        @test step(s) == 1
+        @test length(s) == 1
+        @test PySlice(s) === s
+        @test s == pybuiltin("slice")(4, 5)
+    end
+
+    @test PySlice(5) == 0:4
+    @test PySlice(1,5) == 1:4
+    @test PySlice(1,6,2) == 1:2:5
+    @test PySlice(1:5) == 1:5
+    @test PySlice(2:2:6) == 2:2:6
+
     # check type stability of pycall with an explicit return type
     @inferred pycall(PyObject(1).__add__, Int, 2)
 
@@ -107,6 +151,57 @@ const PyInt = pyversion < v"3" ? Int : Clonglong
                 @test GC.@preserve(o, pyincref.(PyArray(o))) == a
             end
         end
+        let A = Float64[1 2; 3 4]
+            # Normal array
+            B = copy(A)
+            C = PyArray( PyObject(B) )
+            @test C == B
+            B[1] = 3
+            @test C == B && C[1] == B[1]
+
+            C = PyArray( PyCall.PyObjectWithParent(B) )
+            @test C == B
+            B[1] = 2
+            @test C == B && C[1] == B[1]
+
+            # SubArray
+            B = view(A, 1:2, 2:2)
+            C = PyArray( PyObject(B) )
+            @test C == B
+            A[3] = 5
+            @test C == B && C[1] == A[3]
+
+            C = PyArray( PyCall.PyObjectWithParent(B) )
+            @test C == B
+            A[3] = 4
+            @test C == B && C[1] == A[3]
+
+            # ReshapedArray
+            B = Base.ReshapedArray( A, (1,4), () )
+            C = PyArray( PyObject(B) )
+            @test C == B
+            A[2] = 6
+            @test C == B && C[2] == A[2]
+
+            # PermutedDimsArray
+            B = PermutedDimsArray(A, (2,1) )
+            C = PyArray( PyObject(B) )
+            @test C == B
+            A[1] == 7
+            @test C == B && C[1] == A[1]
+
+            C = PyArray( PyCall.PyObjectWithParent(B) )
+            @test C == B
+            A[1] == 8
+            @test C == B && C[1] == A[1]
+
+            # ReinterpretArray
+            B = reinterpret(UInt64, A)
+            C = PyArray( PyObject(B) )
+            @test C == B
+            A[1] = 12
+            @test C == B && C[1] == reinterpret(UInt64, A[1])
+        end
     end
     @test PyVector(PyObject([1,3.2,"hello",true])) == [1,3.2,"hello",true]
     @test PyDict(PyObject(Dict(1 => "hello", 2 => "goodbye"))) == Dict(1 => "hello", 2 => "goodbye")