diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index d24cb7637..42206349b 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -12,8 +12,17 @@ steps:
     agents:
       queue: "juliagpu"
       cuda: "*"
+    commands: |
+      julia --project=test -e '
+          if VERSION < v"1.7"
+            using Pkg
+            Pkg.rm("ReTestItems") # not compatible with 1.6
+          end'
     env:
-      GROUP: "CUDA"
+      GROUP: "All"
+      TEST_FAST: true
+      RETESTITEMS_NWORKERS: 2
+      # JULIA_CUDA_HARD_MEMORY_LIMIT: "50%"
     if: build.message !~ /\[skip tests\]/
     timeout_in_minutes: 180
     matrix:
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index 53d936c62..8e5b37afb 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -24,11 +24,15 @@ jobs:
   test:
     name: ${{ matrix.suite }} - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
+    env:
+      GROUP: ${{ matrix.suite }}
+      TEST_FAST: ${{ matrix.version != '1' || matrix.os != 'ubuntu-latest' }}
     strategy:
       fail-fast: false
       matrix:
         version:
-          - '1' # Replace this with the minimum Julia version that your package supports.
+          - '1.6' # Replace this with the minimum Julia version that your package supports.
+          - '1'
           - 'nightly'
         os:
           - ubuntu-latest
@@ -37,18 +41,19 @@ jobs:
         arch:
           - x64
         suite:
-          - '["AlexNet", "VGG"]'
-          - '["GoogLeNet", "SqueezeNet", "MobileNets"]'
-          - '"EfficientNet"'
-          - 'r"/*/ResNet*"'
-          - 'r"/*/SEResNet*"'
-          - '[r"Res2Net", r"Res2NeXt"]'
-          - '"Inception"'
-          - '"DenseNet"'
-          - '"UNet"'
-          - '["ConvNeXt", "ConvMixer"]'
-          - 'r"Mixers"'
-          - 'r"ViTs"'
+          - 'AlexNet|VGG'
+          - 'GoogLeNet|SqueezeNet|MobileNet|MNASNet'
+          - 'EfficientNet'
+          - '^ResNet|WideResNet'
+          - '^ResNeXt' # split off from ResNet to reduce overall runtime
+          - 'SEResNet|SEResNeXt'
+          - 'Res2Net|Res2NeXt'
+          - 'Inception'
+          - 'DenseNet'
+          - 'UNet'
+          - 'ConvNeXt|ConvMixer'
+          - 'MLP-Mixer|ResMLP|gMLP'
+          - 'ViT'
     steps:
       - uses: actions/checkout@v3
       - uses: julia-actions/setup-julia@v1
@@ -57,20 +62,16 @@ jobs:
           arch: ${{ matrix.arch }}
       - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-buildpkg@v1
-      - name: "Setup environment"
-        run: |
-          julia --project=./test -e 'using Pkg; Pkg.develop(path = ".")'
-      - name: "Run tests + coverage"
-        if: matrix.version == '1' && matrix.os == 'ubuntu-latest'
-        run: |
-          julia --code-coverage=user --color=yes --depwarn=yes --project=./test -e 'include("test/retest.jl"); retest(${{ matrix.suite }})'
-        shell: bash
-      - name: "Run tests only"
-        if: ${{ !(matrix.version == '1' && matrix.os == 'ubuntu-latest') }}
+
+      - name: Setup test env for 1.6
+        if: ${{ matrix.version == '1.6' }}
         run: |
-          julia --color=yes --depwarn=yes --project=./test -e 'include("test/retest.jl"); retest(${{ matrix.suite }})'
-        continue-on-error: ${{ matrix.version == 'nightly' }}
-        shell: bash
+          julia --color=yes --depwarn=yes --project=./test -e 'using Pkg; Pkg.rm("ReTestItems")'
+      - name: Run tests
+        uses: julia-actions/julia-runtest@v1
+        continue-on-error: ${{ !(matrix.version == '1' && matrix.os == 'ubuntu-latest') && matrix.version == 'nightly' }}
+        with:
+          coverage: ${{ matrix.version == '1' && matrix.os == 'ubuntu-latest' }}
       - uses: actions/upload-artifact@v3
         with:
           name: coverage-${{ hashFiles('**/*.cov') }}
@@ -78,7 +79,7 @@ jobs:
         if: matrix.version == '1' && matrix.os == 'ubuntu-latest'
 
   coverage:
-    name: "Coverage"
+    name: Coverage
     runs-on: ubuntu-latest
     needs: test
     steps:
@@ -94,7 +95,7 @@ jobs:
           cp -r coverage-*/* .
           rm -rf coverage-*
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v2
+      - uses: codecov/codecov-action@v3
         with:
           file: lcov.info
 
diff --git a/test/Project.toml b/test/Project.toml
index b121720e7..2a93eb04f 100644
--- a/test/Project.toml
+++ b/test/Project.toml
@@ -2,5 +2,18 @@
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0"
-ReTest = "e0db7c4e-2690-44b9-bad6-7687da720f89"
+ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TestImages = "5e47fb64-e119-507b-a336-dd2b206d9990"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
+cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
+
+[compat]
+CUDA = "4, 5"
+Flux = "0.13.16, 0.14"
+Images = "0.26"
+ReTestItems = "1"
+Test = "1"
+TestImages = "1.8"
+TestItemRunner = "0.2"
+cuDNN = "1"
diff --git a/test/convnet_tests.jl b/test/convnet_tests.jl
new file mode 100644
index 000000000..630de19e3
--- /dev/null
+++ b/test/convnet_tests.jl
@@ -0,0 +1,376 @@
+@testitem "AlexNet" setup=[TestModels] begin
+    model = AlexNet() |> gpu
+    @test size(model(x_256)) == (1000, 1)
+    @test_throws ArgumentError AlexNet(pretrain = true)
+    @test gradtest(model, x_256)
+    _gc()
+end
+
+@testitem "VGG" setup=[TestModels] begin
+    sizes = TEST_FAST ? [11] : [11, 13, 16, 19]
+    @testset "VGG($sz, batchnorm=$bn)" for sz in sizes, bn in [true, false]
+        m = VGG(sz; batchnorm = bn) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (VGG, sz, bn) in PRETRAINED_MODELS
+            @test acctest(VGG(sz; batchnorm = bn, pretrain = true))
+        else
+            @test_throws ArgumentError VGG(sz, batchnorm = bn, pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "ResNet" setup=[TestModels] begin
+    # Tests for pretrained ResNets
+    sizes = TEST_FAST ? [18] : [18, 34, 50, 101, 152]
+    @testset "ResNet($sz)" for sz in sizes
+        m = ResNet(sz) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (ResNet, sz) in PRETRAINED_MODELS
+            @test acctest(ResNet(sz; pretrain = true))
+        else
+            @test_throws ArgumentError ResNet(sz, pretrain = true)
+        end
+    end
+
+    @testset "resnet" begin
+        @testset for block_fn in [Metalhead.basicblock, Metalhead.bottleneck]
+            layer_list = [
+                [2, 2, 2, 2],
+                [3, 4, 6, 3],
+                [3, 4, 23, 3],
+                [3, 8, 36, 3],
+            ]
+            @testset for layers in layer_list
+                drop_list = [
+                    (dropout_prob = 0.1, stochastic_depth_prob = 0.1, dropblock_prob = 0.1),
+                    (dropout_prob = 0.5, stochastic_depth_prob = 0.5, dropblock_prob = 0.5),
+                    (dropout_prob = 0.8, stochastic_depth_prob = 0.8, dropblock_prob = 0.8),
+                ]
+                @testset for drop_probs in drop_list
+                    m = Metalhead.resnet(block_fn, layers; drop_probs...) |> gpu
+                    @test size(m(x_224)) == (1000, 1)
+                    @test gradtest(m, x_224)
+                    _gc()
+                end
+            end
+        end
+    end
+end
+
+@testitem "WideResNet" setup=[TestModels] begin
+    sizes = TEST_FAST ? [50] : [50, 101]
+    @testset "WideResNet($sz)" for sz in sizes
+        m = WideResNet(sz) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        @test gradtest(m, x_224)
+        _gc()
+        if (WideResNet, sz) in PRETRAINED_MODELS
+            @test acctest(WideResNet(sz; pretrain = true))
+        else
+            @test_throws ArgumentError WideResNet(sz, pretrain = true)
+        end
+    end
+end
+
+@testitem "ResNeXt" setup=[TestModels] begin
+    depths = TEST_FAST ? [50] : [50, 101, 152]
+    cardinalities = TEST_FAST ? [32] : [32, 64]
+    base_widths = TEST_FAST ? [4] : [4, 8]
+    @testset for depth in depths, cardinality in cardinalities, base_width in base_widths
+        m = ResNeXt(depth; cardinality, base_width) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (ResNeXt, depth, cardinality, base_width) in PRETRAINED_MODELS
+            @test acctest(ResNeXt(depth; cardinality, base_width, pretrain = true))
+        else
+            @test_throws ArgumentError ResNeXt(depth;
+                cardinality,
+                base_width,
+                pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "SEResNet" setup=[TestModels] begin
+    depths = TEST_FAST ? [18] : [18, 34, 50, 101, 152]
+    @testset for depth in depths
+        m = SEResNet(depth) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (SEResNet, depth) in PRETRAINED_MODELS
+            @test acctest(SEResNet(depth; pretrain = true))
+        else
+            @test_throws ArgumentError SEResNet(depth, pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "SEResNeXt" setup=[TestModels] begin
+    depths = TEST_FAST ? [50] : [50, 101, 152]
+    cardinalities = TEST_FAST ? [32] : [32, 64]
+    base_widths = TEST_FAST ? [4] : [4, 8]
+    @testset for depth in depths, cardinality in cardinalities, base_width in base_widths
+        m = SEResNeXt(depth; cardinality, base_width) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (SEResNeXt, depth, cardinality, base_width) in PRETRAINED_MODELS
+            @test acctest(SEResNeXt(depth; pretrain = true))
+        else
+            @test_throws ArgumentError SEResNeXt(depth, pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "Res2Net" setup=[TestModels] begin
+    configs = TEST_FAST ? [(26, 4)] : [(26, 4), (48, 2), (14, 8), (26, 6), (26, 8)]
+    @testset for (base_width, scale) in configs
+        m = Res2Net(50; base_width, scale) |> gpu
+        if has_cuda()
+            @test_broken size(m(x_224)) == (1000, 1)
+        else
+            @test size(m(x_224)) == (1000, 1)
+        end
+        if (Res2Net, 50, base_width, scale) in PRETRAINED_MODELS
+            if has_cuda()
+                @test_broken acctest(Res2Net(50; base_width, scale, pretrain = true))
+            else
+                @test acctest(Res2Net(50; base_width, scale, pretrain = true))
+            end
+        else
+            err_type = VERSION < v"1.7" && has_cuda() ? Exception : ArgumentError
+            @test_throws err_type Res2Net(50; base_width, scale, pretrain = true)
+        end
+        if has_cuda()
+            @test_broken gradtest(m, x_224)
+        else
+            @test gradtest(m, x_224)
+        end
+        _gc()
+    end
+
+    if !TEST_FAST
+        @testset for (base_width, scale) in [(26, 4)]
+            m = Res2Net(101; base_width, scale) |> gpu
+            @test size(m(x_224)) == (1000, 1)
+            if (Res2Net, 101, base_width, scale) in PRETRAINED_MODELS
+                @test acctest(Res2Net(101; base_width, scale, pretrain = true))
+            else
+                @test_throws ArgumentError Res2Net(101; base_width, scale, pretrain = true)
+            end
+            @test gradtest(m, x_224)
+            _gc()
+        end
+    end
+end
+
+@testitem "Res2NeXt" setup=[TestModels] begin
+    @testset for depth in [50, 101]
+        m = Res2NeXt(depth) |> gpu
+        if has_cuda()
+            @test_broken size(m(x_224)) == (1000, 1)
+        else
+            @test size(m(x_224)) == (1000, 1)
+        end
+        if (Res2NeXt, depth) in PRETRAINED_MODELS
+            @test acctest(Res2NeXt(depth; pretrain = true))
+        else
+            @test_throws ArgumentError Res2NeXt(depth, pretrain = true)
+        end
+        if has_cuda()
+            @test_broken gradtest(m, x_224)
+        else
+            @test gradtest(m, x_224)
+        end
+        _gc()
+    end
+end
+
+@testitem "EfficientNet" setup=[TestModels] begin
+    configs = TEST_FAST ? [:b0] : [:b0, :b1, :b2, :b3, :b4, :b5] #:b6, :b7, :b8]
+    @testset "EfficientNet($config)" for config in configs
+        # preferred image resolution scaling
+        r = Metalhead.EFFICIENTNET_GLOBAL_CONFIGS[config][1]
+        x = rand(Float32, r, r, 3, 1) |> gpu
+        m = EfficientNet(config) |> gpu
+        @test size(m(x)) == (1000, 1)
+        if (EfficientNet, config) in PRETRAINED_MODELS
+            @test acctest(EfficientNet(config; pretrain = true))
+        else
+            @test_throws ArgumentError EfficientNet(config, pretrain = true)
+        end
+        @test gradtest(m, x)
+        _gc()
+    end
+end
+
+@testitem "EfficientNetv2" setup=[TestModels] begin
+    configs = TEST_FAST ? [:small] : [:small, :medium, :large] # :xlarge]
+    @testset for config in configs
+        m = EfficientNetv2(config) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (EfficientNetv2, config) in PRETRAINED_MODELS
+            @test acctest(EfficientNetv2(config; pretrain = true))
+        else
+            @test_throws ArgumentError EfficientNetv2(config, pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "GoogLeNet" setup=[TestModels] begin
+    @testset for bn in [true, false]
+        m = GoogLeNet(; batchnorm = bn) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (GoogLeNet, bn) in PRETRAINED_MODELS
+            @test acctest(GoogLeNet(; batchnorm = bn, pretrain = true))
+        else
+            @test_throws ArgumentError GoogLeNet(batchnorm = bn, pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "Inception" setup=[TestModels] begin
+    x_299 = rand(Float32, 299, 299, 3, 2) |> gpu
+    @testset "$Model" for Model in [Inceptionv3, Inceptionv4, InceptionResNetv2, Xception]
+        m = Model() |> gpu
+        @test size(m(x_299)) == (1000, 2)
+        if Model in PRETRAINED_MODELS
+            @test acctest(Model(; pretrain = true))
+        else
+            @test_throws ArgumentError Model(pretrain = true)
+        end
+        @test gradtest(m, x_299)
+        _gc()
+    end
+end
+
+@testitem "SqueezeNet" setup=[TestModels] begin
+    m = SqueezeNet() |> gpu
+    @test size(m(x_224)) == (1000, 1)
+    if SqueezeNet in PRETRAINED_MODELS
+        if VERSION >= v"1.7"
+            @test acctest(SqueezeNet(; pretrain = true))
+        else
+            @test_broken acctest(SqueezeNet(; pretrain = true))
+        end
+    else
+        @test_throws ArgumentError SqueezeNet(pretrain = true)
+    end
+    @test gradtest(m, x_224)
+    _gc()
+end
+
+@testitem "DenseNet" setup=[TestModels] begin
+    sizes = TEST_FAST ? [121] : [121, 161, 169, 201]
+    @testset for sz in sizes
+        m = DenseNet(sz) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (DenseNet, sz) in PRETRAINED_MODELS
+            @test acctest(DenseNet(sz; pretrain = true))
+        else
+            @test_throws ArgumentError DenseNet(sz, pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testsetup module TestMobileNets
+export WIDTH_MULTS
+const WIDTH_MULTS = get(ENV, "FAST_TEST", "false") == "true" ? [0.5] : [0.5, 0.75, 1.0, 1.3]
+end
+
+@testitem "MobileNetV1" setup=[TestModels, TestMobileNets] begin
+    @testset for width_mult in WIDTH_MULTS
+        m = MobileNetv1(width_mult) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (MobileNetv1, width_mult) in PRETRAINED_MODELS
+            @test acctest(MobileNetv1(; pretrain = true))
+        else
+            @test_throws ArgumentError MobileNetv1(pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "MobileNetv2" setup=[TestModels, TestMobileNets] begin
+    @testset for width_mult in WIDTH_MULTS
+        m = MobileNetv2(width_mult) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (MobileNetv2, width_mult) in PRETRAINED_MODELS
+            @test acctest(MobileNetv2(; pretrain = true))
+        else
+            @test_throws ArgumentError MobileNetv2(pretrain = true)
+        end
+        @test gradtest(m, x_224)
+    end
+end
+
+@testitem "MobileNetv3" setup=[TestModels, TestMobileNets] begin
+    configs = TEST_FAST ? [:small] : [:small, :large]
+    @testset for width_mult in WIDTH_MULTS, config in configs
+        m = MobileNetv3(config; width_mult) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (MobileNetv3, config, width_mult) in PRETRAINED_MODELS
+            @test acctest(MobileNetv3(config; pretrain = true))
+        else
+            @test_throws ArgumentError MobileNetv3(config; pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "MNASNet" setup=[TestModels, TestMobileNets] begin
+    @testset for width_mult in WIDTH_MULTS, config in [:A1, :B1]
+        m = MNASNet(config; width_mult) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        if (MNASNet, config, width_mult) in PRETRAINED_MODELS
+            @test acctest(MNASNet(config; pretrain = true))
+        else
+            @test_throws ArgumentError MNASNet(config; pretrain = true)
+        end
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "ConvNeXt" setup=[TestModels] begin
+    configs = TEST_FAST ? [:small] : [:small, :base, :large, :tiny, :xlarge]
+    @testset for config in configs
+        m = ConvNeXt(config) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "ConvMixer" setup=[TestModels] begin
+    configs = TEST_FAST ? [:small] : [:small, :base, :large]
+    @testset for config in configs
+        m = ConvMixer(config) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "UNet" setup=[TestModels] begin
+    encoder = Metalhead.backbone(ResNet(18))
+    model = UNet((256, 256), 3, 10, encoder) |> gpu
+    @test size(model(x_256)) == (256, 256, 10, 1)
+    @test gradtest(model, x_256)
+
+    model = UNet() |> gpu
+    @test size(model(x_256)) == (256, 256, 3, 1)
+    _gc()
+end
diff --git a/test/convnets.jl b/test/convnets.jl
deleted file mode 100644
index 774a44b25..000000000
--- a/test/convnets.jl
+++ /dev/null
@@ -1,361 +0,0 @@
-@testset "AlexNet" begin
-    model = AlexNet()
-    @test size(model(x_256)) == (1000, 1)
-    @test_throws ArgumentError AlexNet(pretrain = true)
-    @test gradtest(model, x_256)
-    _gc()
-end
-
-@testset "VGG" begin
-    @testset "VGG($sz, batchnorm=$bn)" for sz in [11, 13, 16, 19], bn in [true, false]
-        m = VGG(sz, batchnorm = bn)
-        @test size(m(x_224)) == (1000, 1)
-        if (VGG, sz, bn) in PRETRAINED_MODELS
-            @test acctest(VGG(sz, batchnorm = bn, pretrain = true))
-        else
-            @test_throws ArgumentError VGG(sz, batchnorm = bn, pretrain = true)
-        end
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
-
-@testset "ResNet" begin
-    # Tests for pretrained ResNets
-    @testset "ResNet($sz)" for sz in [18, 34, 50, 101, 152]
-        m = ResNet(sz)
-        @test size(m(x_224)) == (1000, 1)
-        if (ResNet, sz) in PRETRAINED_MODELS
-            @test acctest(ResNet(sz, pretrain = true))
-        else
-            @test_throws ArgumentError ResNet(sz, pretrain = true)
-        end
-    end
-
-    @testset "resnet" begin
-        @testset for block_fn in [Metalhead.basicblock, Metalhead.bottleneck]
-            layer_list = [
-                [2, 2, 2, 2],
-                [3, 4, 6, 3],
-                [3, 4, 23, 3],
-                [3, 8, 36, 3]
-            ]
-            @testset for layers in layer_list
-                drop_list = [
-                    (dropout_prob = 0.1, stochastic_depth_prob = 0.1, dropblock_prob = 0.1),
-                    (dropout_prob = 0.5, stochastic_depth_prob = 0.5, dropblock_prob = 0.5),
-                    (dropout_prob = 0.8, stochastic_depth_prob = 0.8, dropblock_prob = 0.8),
-                ]
-                @testset for drop_probs in drop_list
-                    m = Metalhead.resnet(block_fn, layers; drop_probs...)
-                    @test size(m(x_224)) == (1000, 1)
-                    @test gradtest(m, x_224)
-                    _gc()
-                end
-            end
-        end
-    end
-end
-
-
-@testset "WideResNet" begin
-    @testset "WideResNet($sz)" for sz in [50, 101]
-        m = WideResNet(sz)
-        @test size(m(x_224)) == (1000, 1)
-        @test gradtest(m, x_224)
-        _gc()
-        if (WideResNet, sz) in PRETRAINED_MODELS
-            @test acctest(WideResNet(sz, pretrain = true))
-        else
-            @test_throws ArgumentError WideResNet(sz, pretrain = true)
-        end
-    end
-end
-
-@testset "ResNeXt" begin
-    @testset for depth in [50, 101, 152]
-        @testset for cardinality in [32, 64]
-            @testset for base_width in [4, 8]
-                m = ResNeXt(depth; cardinality, base_width)
-                @test size(m(x_224)) == (1000, 1)
-                if (ResNeXt, depth, cardinality, base_width) in PRETRAINED_MODELS
-                    @test acctest(ResNeXt(depth; cardinality, base_width, pretrain = true))
-                else
-                    @test_throws ArgumentError ResNeXt(depth; cardinality, base_width, pretrain = true)
-                end
-                @test gradtest(m, x_224)
-                _gc()
-            end
-        end
-    end
-end
-
-@testset "SEResNet" begin
-    @testset for depth in [18, 34, 50, 101, 152]
-        m = SEResNet(depth)
-        @test size(m(x_224)) == (1000, 1)
-        if (SEResNet, depth) in PRETRAINED_MODELS
-            @test acctest(SEResNet(depth, pretrain = true))
-        else
-            @test_throws ArgumentError SEResNet(depth, pretrain = true)
-        end
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
-
-@testset "SEResNeXt" begin
-    @testset for depth in [50, 101, 152]
-        @testset for cardinality in [32, 64]
-            @testset for base_width in [4, 8]
-                m = SEResNeXt(depth; cardinality, base_width)
-                @test size(m(x_224)) == (1000, 1)
-                if (SEResNeXt, depth, cardinality, base_width) in PRETRAINED_MODELS
-                    @test acctest(SEResNeXt(depth, pretrain = true))
-                else
-                    @test_throws ArgumentError SEResNeXt(depth, pretrain = true)
-                end
-                @test gradtest(m, x_224)
-                _gc()
-            end
-        end
-    end
-end
-
-@testset "Res2Net" begin
-    @testset for (base_width, scale) in [(26, 4), (48, 2), (14, 8), (26, 6), (26, 8)]
-        m = Res2Net(50; base_width, scale)
-        @test size(m(x_224)) == (1000, 1)
-        if (Res2Net, 50, base_width, scale) in PRETRAINED_MODELS
-            @test acctest(Res2Net(50; base_width, scale, pretrain = true))
-        else
-            @test_throws ArgumentError Res2Net(50; base_width, scale, pretrain = true)
-        end
-        @test gradtest(m, x_224)
-        _gc()
-    end
-    @testset for (base_width, scale) in [(26, 4)]
-        m = Res2Net(101; base_width, scale)
-        @test size(m(x_224)) == (1000, 1)
-        if (Res2Net, 101, base_width, scale) in PRETRAINED_MODELS
-            @test acctest(Res2Net(101; base_width, scale, pretrain = true))
-        else
-            @test_throws ArgumentError Res2Net(101; base_width, scale, pretrain = true)
-        end
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
-
-@testset "Res2NeXt" begin
-    @testset for depth in [50, 101]
-        m = Res2NeXt(depth)
-        @test size(m(x_224)) == (1000, 1)
-        if (Res2NeXt, depth) in PRETRAINED_MODELS
-            @test acctest(Res2NeXt(depth, pretrain = true))
-        else
-            @test_throws ArgumentError Res2NeXt(depth, pretrain = true)
-        end
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
-
-@testset "EfficientNet" begin
-    @testset "EfficientNet($config)" for config in [:b0, :b1, :b2, :b3, :b4, :b5,] #:b6, :b7, :b8]
-        # preferred image resolution scaling
-        r = Metalhead.EFFICIENTNET_GLOBAL_CONFIGS[config][1]
-        x = rand(Float32, r, r, 3, 1)
-        m = EfficientNet(config)
-        @test size(m(x)) == (1000, 1)
-        if (EfficientNet, config) in PRETRAINED_MODELS
-            @test acctest(EfficientNet(config, pretrain = true))
-        else
-            @test_throws ArgumentError EfficientNet(config, pretrain = true)
-        end
-        @test gradtest(m, x)
-        _gc()
-    end
-end
-
-@testset "EfficientNetv2" begin
-    @testset for config in [:small, :medium, :large] # :xlarge]
-        m = EfficientNetv2(config)
-        @test size(m(x_224)) == (1000, 1)
-        if (EfficientNetv2, config) in PRETRAINED_MODELS
-            @test acctest(EfficientNetv2(config, pretrain = true))
-        else
-            @test_throws ArgumentError EfficientNetv2(config, pretrain = true)
-        end
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
-
-@testset "GoogLeNet" begin
-    @testset for bn in [true, false]
-        m = GoogLeNet(batchnorm = bn)
-        @test size(m(x_224)) == (1000, 1)
-        if (GoogLeNet, bn) in PRETRAINED_MODELS
-            @test acctest(GoogLeNet(batchnorm = bn, pretrain = true))
-        else
-            @test_throws ArgumentError GoogLeNet(batchnorm = bn, pretrain = true)
-        end
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
-
-@testset "Inception" begin
-    x_299 = rand(Float32, 299, 299, 3, 2)
-    @testset "Inceptionv3" begin
-        m = Inceptionv3()
-        @test size(m(x_299)) == (1000, 2)
-        if Inceptionv3 in PRETRAINED_MODELS
-            @test acctest(Inceptionv3(pretrain = true))
-        else
-            @test_throws ArgumentError Inceptionv3(pretrain = true)
-        end
-        @test gradtest(m, x_299)
-    end
-    _gc()
-    @testset "Inceptionv4" begin
-        m = Inceptionv4()
-        @test size(m(x_299)) == (1000, 2)
-        if Inceptionv4 in PRETRAINED_MODELS
-            @test acctest(Inceptionv4(pretrain = true))
-        else
-            @test_throws ArgumentError Inceptionv4(pretrain = true)
-        end
-        @test gradtest(m, x_299)
-    end
-    _gc()
-    @testset "InceptionResNetv2" begin
-        m = InceptionResNetv2()
-        @test size(m(x_299)) == (1000, 2)
-        if InceptionResNetv2 in PRETRAINED_MODELS
-            @test acctest(InceptionResNetv2(pretrain = true))
-        else
-            @test_throws ArgumentError InceptionResNetv2(pretrain = true)
-        end
-        @test gradtest(m, x_299)
-    end
-    _gc()
-    @testset "Xception" begin
-        m = Xception()
-        @test size(m(x_299)) == (1000, 2)
-        if Xception in PRETRAINED_MODELS
-            @test acctest(Xception(pretrain = true))
-        else
-            @test_throws ArgumentError Xception(pretrain = true)
-        end
-        @test gradtest(m, x_299)
-    end
-    _gc()
-end
-
-@testset "SqueezeNet" begin
-    m = SqueezeNet()
-    @test size(m(x_224)) == (1000, 1)
-    if SqueezeNet in PRETRAINED_MODELS
-        @test acctest(SqueezeNet(pretrain = true))
-    else
-        @test_throws ArgumentError SqueezeNet(pretrain = true)
-    end
-    @test gradtest(m, x_224)
-    _gc()
-end
-
-@testset "DenseNet" begin
-    @testset for sz in [121, 161, 169, 201]
-        m = DenseNet(sz)
-        @test size(m(x_224)) == (1000, 1)
-        if (DenseNet, sz) in PRETRAINED_MODELS
-            @test acctest(DenseNet(sz, pretrain = true))
-        else
-            @test_throws ArgumentError DenseNet(sz, pretrain = true)
-        end
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
-
-@testset "MobileNets (width = $width_mult)" for width_mult in [0.5, 0.75, 1, 1.3]
-    @testset "MobileNetv1" begin
-        m = MobileNetv1(width_mult)
-        @test size(m(x_224)) == (1000, 1)
-        if (MobileNetv1, width_mult) in PRETRAINED_MODELS
-            @test acctest(MobileNetv1(pretrain = true))
-        else
-            @test_throws ArgumentError MobileNetv1(pretrain = true)
-        end
-        @test gradtest(m, x_224)
-    end
-    _gc()
-    @testset "MobileNetv2" begin
-        m = MobileNetv2(width_mult)
-        @test size(m(x_224)) == (1000, 1)
-        if (MobileNetv2, width_mult) in PRETRAINED_MODELS
-            @test acctest(MobileNetv2(pretrain = true))
-        else
-            @test_throws ArgumentError MobileNetv2(pretrain = true)
-        end
-        @test gradtest(m, x_224)
-    end
-    _gc()
-    @testset "MobileNetv3" verbose = true begin
-        @testset for config in [:small, :large]
-            m = MobileNetv3(config; width_mult)
-            @test size(m(x_224)) == (1000, 1)
-            if (MobileNetv3, config, width_mult) in PRETRAINED_MODELS
-                @test acctest(MobileNetv3(config; pretrain = true))
-            else
-                @test_throws ArgumentError MobileNetv3(config; pretrain = true)
-            end
-            @test gradtest(m, x_224)
-            _gc()
-        end
-    end
-    @testset "MNASNet" verbose = true begin
-        @testset for config in [:A1, :B1]
-            m = MNASNet(config; width_mult)
-            @test size(m(x_224)) == (1000, 1)
-            if (MNASNet, config, width_mult) in PRETRAINED_MODELS
-                @test acctest(MNASNet(config; pretrain = true))
-            else
-                @test_throws ArgumentError MNASNet(config; pretrain = true)
-            end
-            @test gradtest(m, x_224)
-            _gc()
-        end
-    end
-end
-
-@testset "ConvNeXt" verbose = true begin
-    @testset for config in [:small, :base, :large, :tiny, :xlarge]
-        m = ConvNeXt(config)
-        @test size(m(x_224)) == (1000, 1)
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
-
-@testset "ConvMixer" verbose = true begin
-    @testset for config in [:small, :base, :large]
-        m = ConvMixer(config)
-        @test size(m(x_224)) == (1000, 1)
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
-
-@testset "UNet" begin
-    encoder = Metalhead.backbone(ResNet(18))
-    model = UNet((256, 256), 3, 10, encoder)
-    @test size(model(x_256)) == (256, 256, 10, 1)
-    @test gradtest(model, x_256)
-
-    model = UNet()
-    @test size(model(x_256)) == (256, 256, 3, 1)
-    _gc()
-end
diff --git a/test/mixer_tests.jl b/test/mixer_tests.jl
new file mode 100644
index 000000000..a6177d88c
--- /dev/null
+++ b/test/mixer_tests.jl
@@ -0,0 +1,34 @@
+@testitem "MLP-Mixer" setup=[TestModels] begin
+    configs = TEST_FAST ? [:small] : [:small, :base, :large]
+    @testset for config in configs
+        m = MLPMixer(config) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "ResMLP" setup=[TestModels] begin
+    configs = TEST_FAST ? [:small] : [:small, :base, :large]
+    @testset for config in configs
+        m = ResMLP(config) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
+
+@testitem "gMLP" setup=[TestModels] begin
+    configs = TEST_FAST ? [:small] : [:small, :base, :large]
+    @testset for config in configs
+        m = gMLP(config) |> gpu
+        if has_cuda()
+            @test_broken size(m(x_224)) == (1000, 1)
+            @test_broken gradtest(m, x_224)
+        else
+            @test size(m(x_224)) == (1000, 1)
+            @test gradtest(m, x_224)
+        end
+        _gc()
+    end
+end
\ No newline at end of file
diff --git a/test/mixers.jl b/test/mixers.jl
deleted file mode 100644
index 2a5d9af70..000000000
--- a/test/mixers.jl
+++ /dev/null
@@ -1,8 +0,0 @@
-@testset for model in [MLPMixer, ResMLP, gMLP]
-    @testset for config in [:small, :base, :large]
-        m = model(config)
-        @test size(m(x_224)) == (1000, 1)
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end
diff --git a/test/model_tests.jl b/test/model_tests.jl
new file mode 100644
index 000000000..729701bc4
--- /dev/null
+++ b/test/model_tests.jl
@@ -0,0 +1,85 @@
+@testsetup module TestModels
+using Metalhead, Images, TestImages
+using Flux: gradient, gpu
+using CUDA: CUDA, has_cuda
+
+export PRETRAINED_MODELS,
+    TEST_FAST,
+    _gc,
+    gradtest,
+    normalize_imagenet,
+    TEST_PATH,
+    TEST_IMG,
+    TEST_X,
+    TEST_LBLS,
+    acctest,
+    x_224,
+    x_256,
+    gpu,
+    has_cuda
+
+const PRETRAINED_MODELS = [
+    # (DenseNet, 121),
+    # (DenseNet, 161),
+    # (DenseNet, 169),
+    # (DenseNet, 201),
+    (ResNet, 18),
+    (ResNet, 34),
+    (ResNet, 50),
+    (ResNet, 101),
+    (ResNet, 152),
+    (ResNeXt, 50, 32, 4),
+    (ResNeXt, 101, 64, 4),
+    (ResNeXt, 101, 32, 8),
+    SqueezeNet,
+    (WideResNet, 50),
+    (WideResNet, 101),
+    (ViT, :base, (16, 16)),
+    (ViT, :base, (32, 32)),
+    (ViT, :large, (16, 16)),
+    (ViT, :large, (32, 32)),
+    (VGG, 11, false),
+    (VGG, 13, false),
+    (VGG, 16, false),
+    (VGG, 19, false),
+]
+
+const TEST_FAST = get(ENV, "TEST_FAST", "false") == "true"
+
+function _gc()
+    GC.safepoint()
+    GC.gc(true)
+    has_cuda() && CUDA.reclaim()
+end
+
+function gradtest(model, input)
+    gradient(sum ∘ model, input)
+    # if we make it to here with no error, success!
+    return true
+end
+
+function normalize_imagenet(data)
+    cmean = reshape(Float32[0.485, 0.456, 0.406], (1, 1, 3, 1))
+    cstd = reshape(Float32[0.229, 0.224, 0.225], (1, 1, 3, 1))
+    return (data .- cmean) ./ cstd
+end
+
+# test image
+const TEST_IMG = imresize(testimage("monarch_color_256"), (224, 224))
+# CHW -> WHC
+const TEST_X = let img_array = convert(Array{Float32}, channelview(TEST_IMG))
+    permutedims(img_array, (3, 2, 1)) |> normalize_imagenet |> gpu
+end
+
+# ImageNet labels
+const TEST_LBLS = readlines(download("https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"))
+
+function acctest(model)
+    ypred = gpu(model)(TEST_X) |> collect |> vec
+    top5 = TEST_LBLS[sortperm(ypred; rev = true)]
+    return "monarch" in top5
+end
+
+const x_224 = rand(Float32, 224, 224, 3, 1) |> gpu
+const x_256 = rand(Float32, 256, 256, 3, 1) |> gpu
+end
\ No newline at end of file
diff --git a/test/runtests.jl b/test/runtests.jl
index 603dac280..2ccdb719d 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,81 +1,27 @@
-using Test, Metalhead
-using Flux
-using Flux: Zygote
-using Images
-
-const PRETRAINED_MODELS = [
-    # (DenseNet, 121),
-    # (DenseNet, 161),
-    # (DenseNet, 169),
-    # (DenseNet, 201),
-    (ResNet, 18),
-    (ResNet, 34),
-    (ResNet, 50),
-    (ResNet, 101),
-    (ResNet, 152),
-    (ResNeXt, 50, 32, 4),
-    (ResNeXt, 101, 64, 4),
-    (ResNeXt, 101, 32, 8),
-    SqueezeNet,
-    (WideResNet, 50),
-    (WideResNet, 101),
-    (ViT, :base, (16, 16)),
-    (ViT, :base, (32, 32)),
-    (ViT, :large, (16, 16)),
-    (ViT, :large, (32, 32)),
-    (VGG, 11, false),
-    (VGG, 13, false),
-    (VGG, 16, false),
-    (VGG, 19, false),
-]
-
-function _gc()
-    GC.safepoint()
-    GC.gc(true)
-end
-
-function gradtest(model, input)
-    y, pb = Zygote.pullback(() -> model(input), Flux.params(model))
-    gs = pb(ones(Float32, size(y)))
-    # if we make it to here with no error, success!
-    return true
-end
-
-function normalize_imagenet(data)
-    cmean = reshape(Float32[0.485, 0.456, 0.406], (1, 1, 3, 1))
-    cstd = reshape(Float32[0.229, 0.224, 0.225], (1, 1, 3, 1))
-    return (data .- cmean) ./ cstd
-end
-
-# test image
-const TEST_PATH = download("https://cdn.pixabay.com/photo/2015/05/07/11/02/guitar-756326_960_720.jpg")
-const TEST_IMG = imresize(Images.load(TEST_PATH), (224, 224))
-# CHW -> WHC
-const TEST_X = permutedims(convert(Array{Float32}, channelview(TEST_IMG)), (3, 2, 1)) |> normalize_imagenet
-
-# ImageNet labels
-const TEST_LBLS = readlines(download("https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"))
-
-function acctest(model)
-    ypred = model(TEST_X) |> vec
-    top5 = TEST_LBLS[sortperm(ypred; rev = true)]
-    return "acoustic guitar" in top5
-end
-
-x_224 = rand(Float32, 224, 224, 3, 1)
-x_256 = rand(Float32, 256, 256, 3, 1)
-
-# CNN tests
-@testset verbose = true "ConvNets" begin
-    include("convnets.jl")
-end
-
-# Mixer tests
-@testset verbose = true "Mixers" begin
-    include("mixers.jl")
+using Metalhead: Metalhead
+using CUDA
+
+# TODO account for GPU tests using name or tag filter
+# TODO write GPU tests!
+const test_group = get(ENV, "GROUP", "All")
+const name_filter = test_group == "All" ? nothing : Regex(test_group)
+
+@static if VERSION >= v"1.7"
+    using ReTestItems
+    if parse(Bool, get(ENV, "CI", "false"))
+        runtests(Metalhead; name = name_filter, verbose_results = true)
+    else
+        # For running locally
+        runtests(Metalhead; name = name_filter)
+    end
+else
+    using TestItemRunner
+    function testitem_filter(ti)
+        return name_filter === nothing || match(name_filter, ti.name) !== nothing
+    end
 end
 
-# ViT tests
-@testset verbose = true "ViTs" begin
-    include("vits.jl")
-end
+# Not sure why this needs to be split into a separate conditional...
+@static if VERSION < v"1.7"
+    @run_package_tests filter = testitem_filter
+end
\ No newline at end of file
diff --git a/test/vit_tests.jl b/test/vit_tests.jl
new file mode 100644
index 000000000..eb9969be1
--- /dev/null
+++ b/test/vit_tests.jl
@@ -0,0 +1,9 @@
+@testitem "ViT" setup=[TestModels] begin
+    configs = TEST_FAST ? [:tiny] : [:tiny, :small, :base, :large, :huge] # :giant, :gigantic]
+    @testset for config in configs
+        m = ViT(config) |> gpu
+        @test size(m(x_224)) == (1000, 1)
+        @test gradtest(m, x_224)
+        _gc()
+    end
+end
diff --git a/test/vits.jl b/test/vits.jl
deleted file mode 100644
index 76a606bc9..000000000
--- a/test/vits.jl
+++ /dev/null
@@ -1,8 +0,0 @@
-@testset "ViT" begin
-    for config in [:tiny, :small, :base, :large, :huge] # :giant, :gigantic]
-        m = ViT(config)
-        @test size(m(x_224)) == (1000, 1)
-        @test gradtest(m, x_224)
-        _gc()
-    end
-end