tests: Overhaul GPU testing, and test linalg operations on GPUs

docs/src/darray.md

@@ -684,6 +684,9 @@ From `Base`:
 From `Random`:
 - `rand!`/`randn!`
 
+From `SparseArrays`:
+- `sprand`
+
 From `Statistics`:
 - `mean`
 - `var`
@@ -694,7 +697,7 @@ From `LinearAlgebra`:
 - `*` (Out-of-place Matrix-(Matrix/Vector) multiply)
 - `mul!` (In-place Matrix-Matrix multiply)
 - `cholesky`/`cholesky!` (In-place/Out-of-place Cholesky factorization)
-- `lu`/`lu!` (In-place/Out-of-place LU factorization (`NoPivot` only))
+- `lu`/`lu!` (In-place/Out-of-place LU factorization (`NoPivot` and `RowMaximum` only))
 
 From `AbstractFFTs`:
 - `fft`/`fft!`

ext/CUDAExt.jl

@@ -111,6 +111,12 @@ struct AllocateUndef{S} end
 (::AllocateUndef{S})(T, dims::Dims{N}) where {S,N} = CuArray{S,N}(undef, dims)
 Dagger.allocate_array_func(::CuArrayDeviceProc, ::Dagger.AllocateUndef{S}) where S = AllocateUndef{S}()
 
+# Indexing
+Base.getindex(arr::CuArray, d::Dagger.ArrayDomain) = arr[Dagger.indexes(d)...]
+
+# Views
+Base.view(A::CuArray{T,N}, p::Dagger.Blocks{N}) where {T,N} = Dagger._view(A, p)
+
 # In-place
 # N.B. These methods assume that later operations will implicitly or
 # explicitly synchronize with their associated stream

ext/IntelExt.jl

@@ -106,6 +106,9 @@ struct AllocateUndef{S} end
 (::AllocateUndef{S})(T, dims::Dims{N}) where {S,N} = oneArray{S,N}(undef, dims)
 Dagger.allocate_array_func(::oneArrayDeviceProc, ::Dagger.AllocateUndef{S}) where S = AllocateUndef{S}()
 
+# Indexing
+Base.getindex(arr::oneArray, d::Dagger.ArrayDomain) = arr[Dagger.indexes(d)...]
+
 # In-place
 # N.B. These methods assume that later operations will implicitly or
 # explicitly synchronize with their associated stream

ext/MetalExt.jl

@@ -94,6 +94,9 @@ struct AllocateUndef{S} end
 (::AllocateUndef{S})(T, dims::Dims{N}) where {S,N} = MtlArray{S,N}(undef, dims)
 Dagger.allocate_array_func(::MtlArrayDeviceProc, ::Dagger.AllocateUndef{S}) where S = AllocateUndef{S}()
 
+# Indexing
+Base.getindex(arr::MtlArray, d::Dagger.ArrayDomain) = arr[Dagger.indexes(d)...]
+
 # In-place
 # N.B. These methods assume that later operations will implicitly or
 # explicitly synchronize with their associated stream

ext/OpenCLExt.jl

@@ -107,6 +107,9 @@ struct AllocateUndef{S} end
 (::AllocateUndef{S})(T, dims::Dims{N}) where {S,N} = CLArray{S,N}(undef, dims)
 Dagger.allocate_array_func(::CLArrayDeviceProc, ::Dagger.AllocateUndef{S}) where S = AllocateUndef{S}()
 
+# Indexing
+Base.getindex(arr::CLArray, d::Dagger.ArrayDomain) = arr[Dagger.indexes(d)...]
+
 # In-place
 # N.B. These methods assume that later operations will implicitly or
 # explicitly synchronize with their associated stream

ext/ROCExt.jl

@@ -108,6 +108,9 @@ struct AllocateUndef{S} end
 (::AllocateUndef{S})(T, dims::Dims{N}) where {S,N} = ROCArray{S,N}(undef, dims)
 Dagger.allocate_array_func(::ROCArrayDeviceProc, ::Dagger.AllocateUndef{S}) where S = AllocateUndef{S}()
 
+# Indexing
+Base.getindex(arr::ROCArray, d::Dagger.ArrayDomain) = arr[Dagger.indexes(d)...]
+
 # In-place
 # N.B. These methods assume that later operations will implicitly or
 # explicitly synchronize with their associated stream

src/Dagger.jl

@@ -7,7 +7,7 @@ import SparseArrays: sprand, SparseMatrixCSC
 import MemPool
 import MemPool: DRef, FileRef, poolget, poolset
 
-import Base: collect, reduce
+import Base: collect, reduce, view
 
 import LinearAlgebra
 import LinearAlgebra: Adjoint, BLAS, Diagonal, Bidiagonal, Tridiagonal, LAPACK, LowerTriangular, PosDefException, Transpose, UpperTriangular, UnitLowerTriangular, UnitUpperTriangular, diagind, ishermitian, issymmetric

src/array/alloc.jl

@@ -167,7 +167,8 @@ function Base.zero(x::DArray{T,N}) where {T,N}
     return _to_darray(a)
 end
 
-function Base.view(A::AbstractArray{T,N}, p::Blocks{N}) where {T,N}
+Base.view(A::AbstractArray{T,N}, p::Blocks{N}) where {T,N} = _view(A, p)
+function _view(A::AbstractArray{T,N}, p::Blocks{N}) where {T,N}
     d = ArrayDomain(Base.index_shape(A))
     dc = partition(p, d)
     # N.B. We use `tochunk` because we only want to take the view locally, and

src/array/lu.jl

@@ -7,8 +7,9 @@ function LinearAlgebra.lu!(A::DMatrix{T}, ::LinearAlgebra.NoPivot; check::Bool=t
     mzone = -one(T)
     Ac = A.chunks
     mt, nt = size(Ac)
-    iscomplex = T <: Complex
-    trans = iscomplex ? 'C' : 'T'
+    mb, nb = A.partitioning.blocksize
+
+    mb != nb && throw(ArgumentError("Unequal block sizes are not supported: mb = $mb, nb = $nb"))
 
     Dagger.spawn_datadeps() do
         for k in range(1, min(mt, nt))
@@ -29,3 +30,95 @@ function LinearAlgebra.lu!(A::DMatrix{T}, ::LinearAlgebra.NoPivot; check::Bool=t
 
     return LinearAlgebra.LU{T,DMatrix{T},DVector{Int}}(A, ipiv, 0)
 end
+
+function searchmax_pivot!(piv_idx::AbstractVector{Int}, piv_val::AbstractVector{T}, A::AbstractMatrix{T}, offset::Int=0) where T
+    max_idx = LinearAlgebra.BLAS.iamax(A[:])
+    piv_idx[1] = offset+max_idx
+    piv_val[1] = A[max_idx]
+end
+
+function update_ipiv!(ipivl::AbstractVector{Int}, piv_idx::AbstractVector{Int}, piv_val::AbstractVector{T}, k::Int, nb::Int) where T
+    max_piv_idx = LinearAlgebra.BLAS.iamax(piv_val)
+    max_piv_val = piv_val[max_piv_idx]
+    abs_max_piv_val = max_piv_val isa Real ? abs(max_piv_val) : abs(real(max_piv_val)) + abs(imag(max_piv_val))
+    isapprox(abs_max_piv_val, zero(T); atol=eps(real(T))) && throw(LinearAlgebra.SingularException(k))
+    ipivl[1] = (max_piv_idx+k-2)*nb + piv_idx[max_piv_idx]
+end
+
+function swaprows_panel!(A::AbstractMatrix{T}, M::AbstractMatrix{T}, ipivl::AbstractVector{Int}, m::Int, p::Int, nb::Int) where T
+    q = div(ipivl[1]-1,nb) + 1
+    r = (ipivl[1]-1)%nb+1
+    if m == q
+        A[p,:], M[r,:] = M[r,:], A[p,:]
+    end
+end
+
+function update_panel!(M::AbstractMatrix{T}, A::AbstractMatrix{T}, p::Int) where T
+    Acinv = one(T) / A[p,p]
+    LinearAlgebra.BLAS.scal!(Acinv, view(M, :, p))
+    LinearAlgebra.BLAS.ger!(-one(T), view(M, :, p), conj.(view(A, p, p+1:size(A,2))), view(M, :, p+1:size(M,2)))
+end
+
+function swaprows_trail!(A::AbstractMatrix{T}, M::AbstractMatrix{T}, ipiv::AbstractVector{Int}, m::Int, nb::Int) where T
+    for p in eachindex(ipiv)
+        q = div(ipiv[p]-1,nb) + 1
+        r = (ipiv[p]-1)%nb+1
+        if m == q
+            A[p,:], M[r,:] = M[r,:], A[p,:]
+        end
+    end
+end
+
+function LinearAlgebra.lu(A::DMatrix{T}, ::LinearAlgebra.RowMaximum; check::Bool=true) where T
+    A_copy = LinearAlgebra._lucopy(A, LinearAlgebra.lutype(T))
+    return LinearAlgebra.lu!(A_copy, LinearAlgebra.RowMaximum(); check=check)
+end
+function LinearAlgebra.lu!(A::DMatrix{T}, ::LinearAlgebra.RowMaximum; check::Bool=true) where T
+    zone = one(T)
+    mzone = -one(T)
+
+    Ac = A.chunks
+    mt, nt = size(Ac)
+    m,  n  = size(A)
+    mb, nb = A.partitioning.blocksize
+
+    mb != nb && throw(ArgumentError("Unequal block sizes are not supported: mb = $mb, nb = $nb"))
+
+    ipiv = DVector(collect(1:min(m, n)), Blocks(mb))
+    ipivc = ipiv.chunks
+
+    max_piv_idx = zeros(Int,mt)
+    max_piv_val = zeros(T, mt)
+
+    Dagger.spawn_datadeps() do
+        for k in 1:min(mt, nt)
+            for p in 1:min(nb, m-(k-1)*nb, n-(k-1)*nb)
+                Dagger.@spawn searchmax_pivot!(Out(view(max_piv_idx, k:k)), Out(view(max_piv_val, k:k)), In(view(Ac[k,k],p:min(nb,m-(k-1)*nb),p:p)), p-1)
+                for i in k+1:mt
+                    Dagger.@spawn searchmax_pivot!(Out(view(max_piv_idx, i:i)), Out(view(max_piv_val, i:i)), In(view(Ac[i,k],:,p:p)))
+                end
+                Dagger.@spawn update_ipiv!(InOut(view(ipivc[k],p:p)), In(view(max_piv_idx, k:mt)), In(view(max_piv_val, k:mt)), k, nb)
+                for i in k:mt
+                    Dagger.@spawn swaprows_panel!(InOut(Ac[k, k]), InOut(Ac[i, k]), In(view(ipivc[k],p:p)), i, p, nb)
+                end
+                Dagger.@spawn update_panel!(InOut(view(Ac[k,k],p+1:min(nb,m-(k-1)*nb),:)), In(Ac[k,k]), p)
+                for i in k+1:mt
+                    Dagger.@spawn update_panel!(InOut(Ac[i, k]), In(Ac[k,k]), p)
+                end
+            end
+            for j in Iterators.flatten((1:k-1, k+1:nt))
+                for i in k:mt
+                    Dagger.@spawn swaprows_trail!(InOut(Ac[k, j]), InOut(Ac[i, j]), In(ipivc[k]), i, mb)
+                end
+            end
+            for j in k+1:nt
+                Dagger.@spawn BLAS.trsm!('L', 'L', 'N', 'U', zone, In(Ac[k, k]), InOut(Ac[k, j]))
+                for i in k+1:mt
+                    Dagger.@spawn BLAS.gemm!('N', 'N', mzone, In(Ac[i, k]), In(Ac[k, j]), zone, InOut(Ac[i, j]))
+                end
+            end
+        end
+    end
+
+    return LinearAlgebra.LU{T,DMatrix{T},DVector{Int}}(A, ipiv, 0)
+end

src/array/mul.jl

@@ -384,9 +384,7 @@ end
     m, n = size(A)
     C = B'
 
-    for i = 1:m, j = 1:n
-        A[i, j] = C[i, j]
-    end
+    A[1:m, 1:n] .= view(C, 1:m, 1:n)
 end
 
 @inline function copydiagtile!(A, uplo)
@@ -401,7 +399,5 @@ end
         C[diagind(C)] .= A[diagind(A)]
     end
 
-    for i = 1:m, j = 1:n
-        A[i, j] = C[i, j]
-    end
+    A[1:m, 1:n] .= view(C, 1:m, 1:n)
 end

test/array/linalg/cholesky.jl

@@ -1,49 +1,60 @@
-@testset "$T" for T in (Float32, Float64, ComplexF32, ComplexF64)
-    D = rand(Blocks(4, 4), T, 32, 32)
-    if !(T <: Complex)
-        @test !issymmetric(D)
-    end
-    @test !ishermitian(D)
+function test_cholesky(AT)
+    @testset "$T" for T in (Float32, Float64, ComplexF32, ComplexF64)
+        D = rand(Blocks(4, 4), T, 32, 32)
+        if !(T <: Complex)
+            @test !issymmetric(D)
+        end
+        @test !ishermitian(D)
 
-    A = rand(T, 128, 128)
-    A = A * A'
-    A[diagind(A)] .+= size(A, 1)
-    B = copy(A)
-    DA = view(A, Blocks(32, 32))
-    if !(T <: Complex)
-        @test issymmetric(DA)
-    end
-    @test ishermitian(DA)
+        A = AT(rand(T, 128, 128))
+        A = A * A'
+        A[diagind(A)] .+= size(A, 1)
+        B = copy(A)
+        DA = view(A, Blocks(32, 32))
+        if !(T <: Complex)
+            @test issymmetric(DA)
+        end
+        @test ishermitian(DA)
 
-    # Out-of-place
-    chol_A = cholesky(A)
-    chol_DA = cholesky(DA)
-    @test chol_DA isa Cholesky
-    @test chol_A.L ≈ chol_DA.L
-    @test chol_A.U ≈ chol_DA.U
-    # Check that cholesky did not modify A or DA
-    @test A ≈ DA ≈ B
+        # Out-of-place
+        chol_A = cholesky(A)
+        chol_DA = cholesky(DA)
+        @test chol_DA isa Cholesky
+        @test chol_A.L ≈ chol_DA.L
+        @test chol_A.U ≈ chol_DA.U
+        # Check that cholesky did not modify A or DA
+        @test A ≈ DA ≈ B
 
-    # In-place
-    A_copy = copy(A)
-    chol_A = cholesky!(A_copy)
-    chol_DA = cholesky!(DA)
-    @test chol_DA isa Cholesky
-    @test chol_A.L ≈ chol_DA.L
-    @test chol_A.U ≈ chol_DA.U
-    # Check that changes propagated to A
-    @test UpperTriangular(collect(DA)) ≈ UpperTriangular(collect(A))
+        # In-place
+        A_copy = copy(A)
+        chol_A = cholesky!(A_copy)
+        chol_DA = cholesky!(DA)
+        @test chol_DA isa Cholesky
+        @test chol_A.L ≈ chol_DA.L
+        @test chol_A.U ≈ chol_DA.U
+        # Check that changes propagated to A
+        @test UpperTriangular(collect(DA)) ≈ UpperTriangular(collect(A))
 
-    # Non-PosDef matrix
-    A = rand(T, 128, 128)
-    A = A * A'
-    A[diagind(A)] .+= size(A, 1)
-    A[1, 1] = -100
-    DA = view(A, Blocks(32, 32))
-    if !(T <: Complex)
-        @test issymmetric(DA)
+        # Non-PosDef matrix
+        A = AT(rand(T, 128, 128))
+        A = A * A'
+        A[diagind(A)] .+= size(A, 1)
+        A[1, 1] = -100
+        DA = view(A, Blocks(32, 32))
+        if !(T <: Complex)
+            @test issymmetric(DA)
+        end
+        @test ishermitian(DA)
+        @test_broken cholesky(DA).U == 42 # This should throw PosDefException
+        #@test_throws_unwrap PosDefException cholesky(DA).U
     end
-    @test ishermitian(DA)
-    @test_broken cholesky(DA).U == 42 # This should throw PosDefException
-    #@test_throws_unwrap PosDefException cholesky(DA).U
 end
+
+for (kind, AT, scope) in ALL_SCOPES
+    kind == :oneAPI || kind == :Metal || kind == :OpenCL && continue
+    @testset "$kind" begin
+        Dagger.with_options(;scope) do
+            test_cholesky(AT)
+        end
+    end
+end

test/array/linalg/core.jl

@@ -1,11 +1,22 @@
-@testset "isapprox" begin
-    A = rand(16, 16)
+function test_linalg_core(AT)
+    @testset "isapprox" begin
+        A = AT(rand(16, 16))
 
-    U1 = UpperTriangular(DArray(A, Blocks(16, 16)))
-    U2 = UpperTriangular(DArray(A, Blocks(16, 16)))
-    @test isapprox(U1, U2)
+        U1 = UpperTriangular(DArray(A, Blocks(16, 16)))
+        U2 = UpperTriangular(DArray(A, Blocks(16, 16)))
+        @test isapprox(U1, U2)
 
-    L1 = LowerTriangular(DArray(A, Blocks(16, 16)))
-    L2 = LowerTriangular(DArray(A, Blocks(16, 16)))
-    @test isapprox(L1, L2)
+        L1 = LowerTriangular(DArray(A, Blocks(16, 16)))
+        L2 = LowerTriangular(DArray(A, Blocks(16, 16)))
+        @test isapprox(L1, L2)
+    end
 end
+
+for (kind, AT, scope) in ALL_SCOPES
+    kind == :oneAPI || kind == :Metal || kind == :OpenCL && continue
+    @testset "$kind" begin
+        Dagger.with_options(;scope) do
+            test_linalg_core(AT)
+        end
+    end
+end