Merge pull request #430 from SciML/ap/testing

Clean up testing

Author: avik-pal

.buildkite/pipeline.yml

@@ -18,7 +18,7 @@ steps:
     if: build.message !~ /\[skip tests\]/
 
 env:
-  GROUP: GPU
+  GROUP: CUDA
   JULIA_PKG_SERVER: "" # it often struggles with our large artifacts
   RETESTITEMS_NWORKERS: 4
   RETESTITEMS_NWORKER_THREADS: 2

.github/workflows/CI.yml

@@ -20,6 +20,8 @@ jobs:
         group:
           - Core
           - Downstream
+          - Misc
+          - Wrappers
         version:
           - "1"
         os:

.github/workflows/FormatCheck.yml

@@ -1,42 +1,9 @@
-name: format-check
+name: Format suggestions
 
-on:
-  push:
-    branches:
-      - 'master'
-      - 'release-'
-    tags: '*'
-  pull_request:
+on: [pull_request]
 
 jobs:
-  build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        julia-version: [1]
-        julia-arch: [x86]
-        os: [ubuntu-latest]
+  code-style:
+    runs-on: ubuntu-latest
     steps:
-      - uses: julia-actions/setup-julia@latest
-        with:
-          version: ${{ matrix.julia-version }}
-
-      - uses: actions/checkout@v4
-      - name: Install JuliaFormatter and format
-        # This will use the latest version by default but you can set the version like so:
-        #
-        # julia  -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter", version="0.13.0"))'
-        run: |
-          julia  -e 'using Pkg; Pkg.add(PackageSpec(name="JuliaFormatter"))'
-          julia  -e 'using JuliaFormatter; format(".", verbose=true)'
-      - name: Format check
-        run: |
-          julia -e '
-          out = Cmd(`git diff --name-only`) |> read |> String
-          if out == ""
-              exit(0)
-          else
-              @error "Some files have not been formatted !!!"
-              write(stdout, out)
-              exit(1)
-          end'
+      - uses: julia-actions/julia-format@v2

Project.toml

@@ -65,7 +65,7 @@ BenchmarkTools = "1.4"
 CUDA = "5.2"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.149.0"
-Enzyme = "0.11.15, 0.12"
+Enzyme = "0.12"
 FastBroadcast = "0.2.8"
 FastClosures = "0.3.2"
 FastLevenbergMarquardt = "0.1"
@@ -76,9 +76,10 @@ LazyArrays = "1.8.2"
 LeastSquaresOptim = "0.8.5"
 LineSearches = "7.2"
 LinearAlgebra = "1.10"
-LinearSolve = "2.29"
+LinearSolve = "2.30"
 MINPACK = "1.2"
 MaybeInplace = "0.1.1"
+ModelingToolkit = "9.13.0"
 NLSolvers = "0.5"
 NLsolve = "4.5"
 NaNMath = "1"
@@ -89,11 +90,10 @@ PrecompileTools = "1.2"
 Preferences = "1.4"
 Printf = "1.10"
 Random = "1.91"
-ReTestItems = "1"
+ReTestItems = "1.24"
 RecursiveArrayTools = "3.8"
 Reexport = "1.2"
 SIAMFANLEquations = "1.0.1"
-SafeTestsets = "0.1"
 SciMLBase = "2.34.0"
 SimpleNonlinearSolve = "1.8"
 SparseArrays = "1.10"
@@ -103,8 +103,8 @@ StableRNGs = "1"
 StaticArrays = "1.9"
 StaticArraysCore = "1.4"
 Sundials = "4.23.1"
-Symbolics = "5.13"
 SymbolicIndexingInterface = "0.3.15"
+Symbolics = "5.26"
 Test = "1.10"
 TimerOutputs = "0.5.23"
 Zygote = "0.6.69"
@@ -115,15 +115,12 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 BandedMatrices = "aae01518-5342-5314-be14-df237901396f"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
-DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
 FastLevenbergMarquardt = "7a0df574-e128-4d35-8cbd-3d84502bf7ce"
 FixedPointAcceleration = "817d07cb-a79a-5c30-9a31-890123675176"
-ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891"
-LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 MINPACK = "4854310b-de5a-5eb6-a2a5-c1dee2bd17f9"
+ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
 NLSolvers = "337daf1e-9722-11e9-073e-8b9effe078ba"
 NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56"
 NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
@@ -132,10 +129,7 @@ OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
-Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SIAMFANLEquations = "084e46ad-d928-497d-ad5e-07fa361a48c4"
-SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
-SparseDiffTools = "47a9eef4-7e08-11e9-0b38-333d64bd3804"
 SpeedMapping = "f1835b91-879b-4a3f-a438-e4baacf14412"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
@@ -145,4 +139,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["Aqua", "Enzyme", "BenchmarkTools", "SafeTestsets", "Pkg", "Test", "ForwardDiff", "StaticArrays", "Symbolics", "LinearSolve", "Random", "LinearAlgebra", "Zygote", "SparseDiffTools", "NonlinearProblemLibrary", "LeastSquaresOptim", "FastLevenbergMarquardt", "NaNMath", "BandedMatrices", "DiffEqBase", "StableRNGs", "MINPACK", "NLsolve", "OrdinaryDiffEq", "SpeedMapping", "FixedPointAcceleration", "SIAMFANLEquations", "Sundials", "ReTestItems", "Reexport", "CUDA", "NLSolvers"]
+test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "FastLevenbergMarquardt", "FixedPointAcceleration", "LeastSquaresOptim", "MINPACK", "ModelingToolkit", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEq", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Symbolics", "Test", "Zygote"]

test/core/23_test_problems_tests.jl

@@ -40,7 +40,7 @@ end
 export test_on_library, problems, dicts
 end
 
-@testitem "PolyAlgorithms" setup=[RobustnessTesting] begin
+@testitem "PolyAlgorithms" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (RobustMultiNewton(), FastShortcutNonlinearPolyalg())
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -50,7 +50,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "NewtonRaphson" setup=[RobustnessTesting] begin
+@testitem "NewtonRaphson" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (NewtonRaphson(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -59,7 +59,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "TrustRegion" setup=[RobustnessTesting] begin
+@testitem "TrustRegion" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Simple),
         TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Fan),
         TrustRegion(; radius_update_scheme = RadiusUpdateSchemes.Hei),
@@ -78,7 +78,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "LevenbergMarquardt" setup=[RobustnessTesting] begin
+@testitem "LevenbergMarquardt" setup=[RobustnessTesting] tags=[:core] begin
     using LinearSolve
 
     alg_ops = (LevenbergMarquardt(), LevenbergMarquardt(; α_geodesic = 0.1),
@@ -92,7 +92,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "DFSane" setup=[RobustnessTesting] begin
+@testitem "DFSane" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (DFSane(),)
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -101,21 +101,28 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "Broyden" setup=[RobustnessTesting] begin
+@testitem "Broyden" setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (Broyden(), Broyden(; init_jacobian = Val(:true_jacobian)),
         Broyden(; update_rule = Val(:bad_broyden)),
         Broyden(; init_jacobian = Val(:true_jacobian), update_rule = Val(:bad_broyden)))
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
-    broken_tests[alg_ops[1]] = [1, 5, 11, 15]
-    broken_tests[alg_ops[2]] = [1, 5, 8, 11, 18]
-    broken_tests[alg_ops[3]] = [1, 5, 9, 11]
-    broken_tests[alg_ops[4]] = [5, 6, 8, 11]
+    if Sys.isapple()
+        broken_tests[alg_ops[1]] = [1, 5, 11]
+        broken_tests[alg_ops[2]] = [1, 5, 8, 11, 18]
+        broken_tests[alg_ops[3]] = [1, 5, 6, 9, 11]
+        broken_tests[alg_ops[4]] = [5, 6, 8, 11]
+    else
+        broken_tests[alg_ops[1]] = [1, 5, 11, 15]
+        broken_tests[alg_ops[2]] = [1, 5, 8, 11, 18]
+        broken_tests[alg_ops[3]] = [1, 5, 9, 11]
+        broken_tests[alg_ops[4]] = [5, 6, 8, 11]
+    end
 
-    test_on_library(problems, dicts, alg_ops, broken_tests)
+    test_on_library(problems, dicts, alg_ops, broken_tests, Sys.isapple() ? 1e-3 : 1e-4)
 end
 
-@testitem "Klement" retries=5 setup=[RobustnessTesting] begin
+@testitem "Klement" retries=5 setup=[RobustnessTesting] tags=[:core] begin
     alg_ops = (Klement(), Klement(; init_jacobian = Val(:true_jacobian_diagonal)))
 
     broken_tests = Dict(alg => Int[] for alg in alg_ops)
@@ -125,7 +132,7 @@ end
     test_on_library(problems, dicts, alg_ops, broken_tests)
 end
 
-@testitem "PseudoTransient" setup=[RobustnessTesting] begin
+@testitem "PseudoTransient" setup=[RobustnessTesting] tags=[:core] begin
     # PT relies on the root being a stable equilibrium for convergence, so it won't work on
     # most problems
     alg_ops = (PseudoTransient(),)

test/core/forward_ad_tests.jl

@@ -62,12 +62,14 @@ __compatible(::KINSOL, ::Val{:oop_cache}) = false
 export test_f!, test_f, jacobian_f, solve_with, __compatible
 end
 
-@testitem "ForwardDiff.jl Integration" setup=[ForwardADTesting] begin
+@testitem "ForwardDiff.jl Integration" setup=[ForwardADTesting] tags=[:core] begin
     for alg in (NewtonRaphson(), TrustRegion(), LevenbergMarquardt(),
         PseudoTransient(; alpha_initial = 10.0), Broyden(), Klement(), DFSane(),
         nothing, NLsolveJL(), CMINPACK(), KINSOL(; globalization_strategy = :LineSearch))
         us = (2.0, @SVector[1.0, 1.0], [1.0, 1.0], ones(2, 2), @SArray ones(2, 2))
 
+        alg isa CMINPACK && Sys.isapple() && continue
+
         @testset "Scalar AD" begin
             for p in 1.0:0.1:100.0, u0 in us, mode in (:iip, :oop, :iip_cache, :oop_cache)
                 __compatible(u0, alg) || continue

test/core/nlls_tests.jl

@@ -48,7 +48,7 @@ end
 export solvers, θ_init, x, y_target, true_function, θ_true, loss_function
 end
 
-@testitem "General NLLS Solvers" setup=[CoreNLLSTesting] begin
+@testitem "General NLLS Solvers" setup=[CoreNLLSTesting] tags=[:core] begin
     prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, x)
     prob_iip = NonlinearLeastSquaresProblem(
         NonlinearFunction(loss_function; resid_prototype = zero(y_target)), θ_init, x)
@@ -62,7 +62,7 @@ end
     end
 end
 
-@testitem "Custom VJP" setup=[CoreNLLSTesting] begin
+@testitem "Custom VJP" setup=[CoreNLLSTesting] tags=[:core] begin
     # This is just for testing that we can use vjp provided by the user
     function vjp(v, θ, p)
         resid = zeros(length(p))
@@ -96,7 +96,7 @@ end
     end
 end
 
-@testitem "NLLS Analytic Jacobian" begin
+@testitem "NLLS Analytic Jacobian" tags=[:core] begin
     dataIn = 1:10
     f(x, p) = x[1] * dataIn .^ 2 .+ x[2] * dataIn .+ x[3]
     dataOut = f([1, 2, 3], nothing) + 0.1 * randn(10, 1)