Sort tests into single and double backends (#245)

* Sort tests into single and double backends

* Fix test deps

* Fix paths

* Fix ENzyme

* DIT

* DataFrame

Author: gdalle

DifferentiationInterface/Project.toml

@@ -80,6 +80,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
 ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
 Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7"
 Tapir = "07d77754-e150-4737-8c94-cd238a1fb45b"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
@@ -104,6 +105,7 @@ test = [
     "PolyesterForwardDiff",
     "ReverseDiff",
     "SparseArrays",
+    "SparseConnectivityTracer",
     "Symbolics",
     "Tapir",
     "Test",

DifferentiationInterface/test/second_order/enzyme_forwarddiff.jl renamed to DifferentiationInterface/test/Double/Enzyme-ForwardDiff.jl

@@ -1,8 +1,32 @@
 using DifferentiationInterface, DifferentiationInterfaceTest
+import DifferentiationInterface as DI
 import DifferentiationInterfaceTest as DIT
 using ForwardDiff: ForwardDiff
 using Zygote: Zygote
 
+backends = [
+    SecondOrder(AutoForwardDiff(), AutoZygote()),
+    MyAutoSparse(SecondOrder(AutoForwardDiff(), AutoZygote())),
+]
+
+for backend in backends
+    @test check_available(backend)
+    @test !check_twoarg(backend)
+    @test check_hessian(backend)
+end
+
+test_differentiation(backends; first_order=false, logging=LOGGING);
+
+test_differentiation(
+    MyAutoSparse(SecondOrder(AutoForwardDiff(), AutoZygote())),
+    sparse_scenarios();
+    first_order=false,
+    sparsity=true,
+    logging=LOGGING,
+);
+
+## Translation
+
 function zygote_breaking_scenarios()
     onearg_scens = filter(default_scenarios()) do scen
         DIT.nb_args(scen) == 1

DifferentiationInterface/test/translation/forwarddiff_zygote.jl renamed to DifferentiationInterface/test/Double/ForwardDiff-Zygote.jl

@@ -16,6 +16,14 @@ end
 
 test_differentiation(backends; second_order=false, logging=LOGGING);
 
+test_differentiation(
+    MyAutoSparse(AutoEnzyme(Enzyme.Reverse)),
+    sparse_scenarios();
+    second_order=false,
+    sparsity=true,
+    logging=LOGGING,
+);
+
 test_differentiation(
     AutoEnzyme(; mode=Enzyme.Forward);  # TODO: add more
     correctness=false,

DifferentiationInterface/test/second_order/reversediff_zygote.jl renamed to DifferentiationInterface/test/Double/ReverseDiff-Zygote.jl

@@ -1,14 +1,12 @@
 using DifferentiationInterface, DifferentiationInterfaceTest
 using FiniteDiff: FiniteDiff
 
-backends = [AutoFiniteDiff()]
-
-for backend in backends
+for backend in [AutoFiniteDiff()]
     @test check_available(backend)
     @test check_twoarg(backend)
     @test check_hessian(backend)
 end
 
 test_differentiation(
-    backends; excluded=[SecondDerivativeScenario, HVPScenario], logging=LOGGING
+    AutoFiniteDiff(); excluded=[SecondDerivativeScenario, HVPScenario], logging=LOGGING
 );

DifferentiationInterface/test/internals/chunk.jl renamed to DifferentiationInterface/test/Internals/chunk.jl

@@ -1,12 +1,14 @@
 using DifferentiationInterface, DifferentiationInterfaceTest
 using FiniteDifferences: FiniteDifferences
 
-backends = [AutoFiniteDifferences(; fdm=FiniteDifferences.central_fdm(3, 1))]
-
-for backend in backends
+for backend in [AutoFiniteDifferences(; fdm=FiniteDifferences.central_fdm(3, 1))]
     @test check_available(backend)
     @test !check_twoarg(backend)
     @test_broken !check_hessian(backend; verbose=false)
 end
 
-test_differentiation(backends; second_order=false, logging=LOGGING);
+test_differentiation(
+    AutoFiniteDifferences(; fdm=FiniteDifferences.central_fdm(3, 1));
+    second_order=false,
+    logging=LOGGING,
+);

DifferentiationInterface/test/internals/coloring.jl renamed to DifferentiationInterface/test/Internals/coloring.jl

@@ -1,11 +1,32 @@
-using DataFrames: DataFrame
 using DifferentiationInterface, DifferentiationInterfaceTest
-import DifferentiationInterface as DI
 using ForwardDiff: ForwardDiff
-using Symbolics: Symbolics
-using Test
+using DataFrames: DataFrame
+
+for backend in [AutoForwardDiff(), AutoSparse(AutoForwardDiff())]
+    @test check_available(backend)
+    @test check_twoarg(backend)
+    @test check_hessian(backend)
+end
+
+test_differentiation([AutoForwardDiff(), AutoSparse(AutoForwardDiff())]; logging=LOGGING);
 
-@testset verbose = false "Dense" begin
+test_differentiation(
+    MyAutoSparse(AutoForwardDiff()), sparse_scenarios(); sparsity=true, logging=LOGGING
+);
+
+test_differentiation(
+    AutoForwardDiff(),
+    # ForwardDiff access individual indices
+    vcat(component_scenarios(), static_scenarios());
+    # jacobian is super slow for some reason
+    excluded=[JacobianScenario],
+    second_order=false,
+    logging=LOGGING,
+);
+
+## Efficiency
+
+@testset verbose = false "Dense efficiency" begin
     # derivative and gradient for `f(x)`
 
     results1 = benchmark_differentiation(
@@ -48,14 +69,10 @@ using Test
     end
 end;
 
-@testset verbose = false "Sparse" begin
+@testset verbose = false "Sparse efficiency" begin
     # sparse jacobian for f!(x, y)
 
-    b_sparse = AutoSparse(
-        AutoForwardDiff(; chunksize=1);
-        sparsity_detector=DI.SymbolicsSparsityDetector(),
-        coloring_algorithm=DI.GreedyColoringAlgorithm(),
-    )
+    b_sparse = MyAutoSparse(AutoForwardDiff(; chunksize=1);)
 
     results1 = benchmark_differentiation(
         [b_sparse],

DifferentiationInterface/test/internals/exceptions.jl renamed to DifferentiationInterface/test/Internals/exceptions.jl

@@ -1,12 +1,10 @@
 using DifferentiationInterface, DifferentiationInterfaceTest
 using PolyesterForwardDiff: PolyesterForwardDiff
 
-backends = [AutoPolyesterForwardDiff(; chunksize=1)]
-
-for backend in backends
+for backend in [AutoPolyesterForwardDiff(; chunksize=1)]
     @test check_available(backend)
     @test check_twoarg(backend)
     @test check_hessian(backend)
 end
 
-test_differentiation(backends; logging=LOGGING);
+test_differentiation(AutoPolyesterForwardDiff(; chunksize=1); logging=LOGGING);

DifferentiationInterface/test/internals/matrices.jl renamed to DifferentiationInterface/test/Internals/matrices.jl

@@ -1,9 +1,7 @@
 using DifferentiationInterface, DifferentiationInterfaceTest
 using Symbolics: Symbolics
 
-backends = [AutoSymbolics(), AutoSparse(AutoSymbolics())]
-
-for backend in backends
+for backend in [AutoSymbolics(), AutoSparse(AutoSymbolics())]
     @test check_available(backend)
     @test check_twoarg(backend)
     @test check_hessian(backend)

DifferentiationInterface/test/first_order/diffractor.jl renamed to DifferentiationInterface/test/Single/Diffractor.jl

@@ -1,12 +1,10 @@
 using DifferentiationInterface, DifferentiationInterfaceTest
 using Tapir: Tapir
 
-backends = [AutoTapir()]
-
-for backend in backends
+for backend in [AutoTapir()]
     @test check_available(backend)
     @test check_twoarg(backend)
     @test !check_hessian(backend; verbose=false)
 end
 
-test_differentiation(backends; second_order=false, logging=LOGGING);
+test_differentiation(AutoTapir(); second_order=false, logging=LOGGING);

DifferentiationInterface/test/first_order/enzyme.jl renamed to DifferentiationInterface/test/Single/Enzyme.jl

@@ -1,12 +1,10 @@
 using DifferentiationInterface, DifferentiationInterfaceTest
 using Tracker: Tracker
 
-backends = [AutoTracker()]
-
-for backend in backends
+for backend in [AutoTracker()]
     @test check_available(backend)
     @test !check_twoarg(backend)
     @test !check_hessian(backend; verbose=false)
 end
 
-test_differentiation(backends; second_order=false, logging=LOGGING);
+test_differentiation(AutoTracker(); second_order=false, logging=LOGGING);

DifferentiationInterface/test/first_order/fastdifferentiation.jl renamed to DifferentiationInterface/test/Single/FastDifferentiation.jl

@@ -1,26 +1,33 @@
 using Pkg
-
 Pkg.develop(
     Pkg.PackageSpec(; path=joinpath(@__DIR__, "..", "..", "DifferentiationInterfaceTest"))
 )
 
+using ADTypes
 using DifferentiationInterface
+using SparseConnectivityTracer: SparseConnectivityTracer
 using Test
 
+function MyAutoSparse(backend::AbstractADType)
+    coloring_algorithm = DifferentiationInterface.GreedyColoringAlgorithm()
+    sparsity_detector = SparseConnectivityTracer.TracerSparsityDetector()
+    return AutoSparse(backend; sparsity_detector, coloring_algorithm)
+end
+
 LOGGING = get(ENV, "CI", "false") == "false"
 
 ## Main tests
 
 @testset verbose = true "DifferentiationInterface.jl" begin
+    @info "Testing formalities"
     @testset verbose = true "Formal tests" begin
         include("formal.jl")
     end
 
-    @testset verbose = true "$folder" for folder in [
-        "first_order", "second_order", "sparse", "translation", "efficiency", "internals"
-    ]
+    @testset verbose = true "$folder" for folder in ["Single", "Double", "Internals"]
         folder_path = joinpath(@__DIR__, folder)
-        @testset verbose = true "$file" for file in readdir(folder_path)
+        @testset verbose = true "$(file[1:end-3])" for file in readdir(folder_path)
+            @info "Testing $folder - $(file[1:end-3])"
             include(joinpath(folder_path, file))
         end
     end