Merge pull request #1604 from lwronski/test-only

Add `testOnly` to filter running test suites

Author: lwronski

modules/cli/src/main/scala/scala/cli/commands/test/Test.scala

@@ -38,7 +38,8 @@ object Test extends ScalaCommand[TestOptions] {
             sharedJava.allJavaOpts.map(JavaOpt(_)).map(Positioned.commandLine)
       ),
       testOptions = baseOptions.testOptions.copy(
-        frameworkOpt = testFramework.map(_.trim).filter(_.nonEmpty)
+        frameworkOpt = testFramework.map(_.trim).filter(_.nonEmpty),
+        testOnly = testOnly.map(_.trim).filter(_.nonEmpty)
       ),
       internalDependencies = baseOptions.internalDependencies.copy(
         addTestRunnerDependencyOpt = Some(true)
@@ -228,11 +229,13 @@ object Test extends ScalaCommand[TestOptions] {
         val testFrameworkOpt0 = testFrameworkOpt.orElse {
           findTestFramework(classPath.map(_.toNIO), logger)
         }
+        val testOnly = build.options.testOptions.testOnly
 
         val extraArgs =
           (if (requireTests) Seq("--require-tests") else Nil) ++
             build.options.internal.verbosity.map(v => s"--verbosity=$v") ++
             testFrameworkOpt0.map(fw => s"--test-framework=$fw").toSeq ++
+            testOnly.map(to => s"--test-only=$to").toSeq ++
             Seq("--") ++ args
 
         Runner.runJvm(

modules/cli/src/main/scala/scala/cli/commands/test/TestOptions.scala

@@ -33,7 +33,12 @@ final case class TestOptions(
   @Group("Test")
   @Tag(tags.should)
   @HelpMessage("Fail if no test suites were run")
-    requireTests: Boolean = false
+    requireTests: Boolean = false,
+  @Group("Test")
+  @Tag(tags.should)
+  @HelpMessage("Specify a glob pattern to filter the tests suite to be run.")
+    testOnly: Option[String] = None
+
 ) extends HasSharedOptions
 // format: on
 

modules/docs-tests/src/main/scala/sclicheck/sclicheck.scala

@@ -145,6 +145,8 @@ def mkBashScript(content: Seq[String]) =
      |${content.mkString("\n")}
      |""".stripMargin
 
+def removeAnsiColors(str: String) = str.replaceAll("\\e\\[[0-9]+m", "")
+
 private lazy val baseTmpDir = {
   val random  = new SecureRandom
   val dirName = s"run-${math.abs(random.nextInt.toLong)}"
@@ -257,7 +259,7 @@ def checkFile(file: os.Path, options: Options): Unit =
 
       case Commands.Check(patterns, regex, _) =>
         check(lastOutput != null, "No output stored from previous commands")
-        val lines = lastOutput.linesIterator.toList
+        val lines = lastOutput.linesIterator.toList.map(removeAnsiColors)
 
         if regex then
           patterns.foreach { pattern =>

modules/integration/src/test/scala/scala/cli/integration/TestTestDefinitions.scala

@@ -258,6 +258,85 @@ abstract class TestTestDefinitions(val scalaVersionOpt: Option[String])
     }
   }
 
+  test("run only one test from munit") {
+    val inputs: TestInputs = TestInputs(
+      os.rel / "MyTests.scala" ->
+        """//> using lib "org.scalameta::munit::0.7.29"
+          |package test
+          |
+          |class MyTests extends munit.FunSuite {
+          |  test("foo") {
+          |    assert(2 + 2 == 5, "foo")
+          |  }
+          |  test("bar") {
+          |    assert(2 + 3 == 5)
+          |    println("Hello from bar")
+          |  }
+          |}
+          |""".stripMargin
+    )
+    inputs.fromRoot { root =>
+      val res =
+        os.proc(
+          TestUtil.cli,
+          "test",
+          ".",
+          extraOptions,
+          "--test-only",
+          "test.MyTests",
+          "--",
+          "*bar*"
+        ).call(cwd = root)
+      val output = res.out.text()
+      expect(res.exitCode == 0)
+      expect(output.contains("Hello from bar"))
+    }
+  }
+  test("run only one test from utest") {
+    val inputs: TestInputs = TestInputs(
+      os.rel / "FooTests.scala" ->
+        """//> using lib "com.lihaoyi::utest::0.7.10"
+          |package tests.foo
+          |import utest._
+          |
+          |object FooTests extends TestSuite {
+          |  val tests = Tests {
+          |    test("foo") {
+          |      assert(2 + 2 == 5)
+          |    }
+          |  }
+          |}
+          |""".stripMargin,
+      os.rel / "BarTests.scala" ->
+        """//> using lib "com.lihaoyi::utest::0.7.10"
+          |package tests.bar
+          |import utest._
+          |
+          |object BarTests extends TestSuite {
+          |  val tests = Tests {
+          |    test("bar") {
+          |      assert(2 + 2 == 4)
+          |      println("Hello from bar")
+          |    }
+          |  }
+          |}
+          |""".stripMargin
+    )
+    inputs.fromRoot { root =>
+      val res =
+        os.proc(
+          TestUtil.cli,
+          "test",
+          ".",
+          extraOptions,
+          "--test-only",
+          "tests.b*"
+        ).call(cwd = root)
+      val output = res.out.text()
+      expect(res.exitCode == 0)
+      expect(output.contains("Hello from bar"))
+    }
+  }
   def successfulNativeTest(): Unit =
     successfulTestInputs.fromRoot { root =>
       val output = os.proc(TestUtil.cli, "test", extraOptions, ".", "--native")

modules/options/src/main/scala/scala/build/options/TestOptions.scala

@@ -1,7 +1,8 @@
 package scala.build.options
 
 final case class TestOptions(
-  frameworkOpt: Option[String] = None
+  frameworkOpt: Option[String] = None,
+  testOnly: Option[String] = None
 )
 
 object TestOptions {

modules/test-runner/src/main/scala/scala/build/testrunner/DynamicTestRunner.scala

@@ -6,6 +6,7 @@ import java.lang.annotation.Annotation
 import java.lang.reflect.Modifier
 import java.nio.file.{Files, Path}
 import java.util.ServiceLoader
+import java.util.regex.Pattern
 
 import scala.annotation.tailrec
 import scala.jdk.CollectionConverters._
@@ -148,27 +149,55 @@ object DynamicTestRunner {
       .headOption
   }
 
+  /** Based on junit-interface [GlobFilter.
+    * compileGlobPattern](https://github.com/sbt/junit-interface/blob/f8c6372ed01ce86f15393b890323d96afbe6d594/src/main/java/com/novocode/junit/GlobFilter.java#L37)
+    *
+    * @return
+    *   Pattern allows to regex input which contains only *, for example `*foo*` match to
+    *   `MyTests.foo`
+    */
+  private def globPattern(expr: String): Pattern = {
+    val a = expr.split("\\*", -1)
+    val b = new StringBuilder()
+    for (i <- 0 until a.length) {
+      if (i != 0) b.append(".*")
+      if (a(i).nonEmpty) b.append(Pattern.quote(a(i).replaceAll("\n", "\\n")))
+    }
+    Pattern.compile(b.toString)
+  }
+
   def main(args: Array[String]): Unit = {
 
-    val (testFrameworkOpt, requireTests, verbosity, args0) = {
+    val (testFrameworkOpt, requireTests, verbosity, testOnly, args0) = {
       @tailrec
       def parse(
         testFrameworkOpt: Option[String],
         reverseTestArgs: List[String],
         requireTests: Boolean,
         verbosity: Int,
+        testOnly: Option[String],
         args: List[String]
-      ): (Option[String], Boolean, Int, List[String]) =
+      ): (Option[String], Boolean, Int, Option[String], List[String]) =
         args match {
-          case Nil => (testFrameworkOpt, requireTests, verbosity, reverseTestArgs.reverse)
+          case Nil => (testFrameworkOpt, requireTests, verbosity, testOnly, reverseTestArgs.reverse)
           case "--" :: t =>
-            (testFrameworkOpt, requireTests, verbosity, reverseTestArgs.reverse ::: t)
+            (testFrameworkOpt, requireTests, verbosity, testOnly, reverseTestArgs.reverse ::: t)
           case h :: t if h.startsWith("--test-framework=") =>
             parse(
               Some(h.stripPrefix("--test-framework=")),
               reverseTestArgs,
               requireTests,
               verbosity,
+              testOnly,
+              t
+            )
+          case h :: t if h.startsWith("--test-only=") =>
+            parse(
+              testFrameworkOpt,
+              reverseTestArgs,
+              requireTests,
+              verbosity,
+              Some(h.stripPrefix("--test-only=")),
               t
             )
           case h :: t if h.startsWith("--verbosity=") =>
@@ -177,14 +206,16 @@ object DynamicTestRunner {
               reverseTestArgs,
               requireTests,
               h.stripPrefix("--verbosity=").toInt,
+              testOnly,
               t
             )
           case "--require-tests" :: t =>
-            parse(testFrameworkOpt, reverseTestArgs, true, verbosity, t)
-          case h :: t => parse(testFrameworkOpt, h :: reverseTestArgs, requireTests, verbosity, t)
+            parse(testFrameworkOpt, reverseTestArgs, true, verbosity, testOnly, t)
+          case h :: t =>
+            parse(testFrameworkOpt, h :: reverseTestArgs, requireTests, verbosity, testOnly, t)
         }
 
-      parse(None, Nil, false, 0, args.toList)
+      parse(None, Nil, false, 0, None, args.toList)
     }
 
     val classLoader = Thread.currentThread().getContextClassLoader
@@ -214,6 +245,12 @@ object DynamicTestRunner {
         .iterator
     }
     val taskDefs = clsFingerprints
+      .filter {
+        case (cls, _) =>
+          testOnly.forall(pattern =>
+            globPattern(pattern).matcher(cls.getName.stripSuffix("$")).matches()
+          )
+      }
       .map {
         case (cls, fp) =>
           new TaskDef(cls.getName.stripSuffix("$"), fp, false, Array(new SuiteSelector))

website/docs/commands/test.md

@@ -102,6 +102,77 @@ MyTests
 foo
 -->
 
+## Filter test suite
+
+Passing the `--test-only` option to the `test` sub-command filters the test suites to be run:
+
+```scala title=BarTests.scala
+//> using lib "org.scalameta::munit::0.7.29"
+package tests.only
+
+class BarTests extends munit.FunSuite {
+  test("bar") {
+    assert(2 + 3 == 5)
+  }
+}
+```
+```scala title=HelloTests.scala
+package tests
+
+class HelloTests extends munit.FunSuite {
+  test("hello") {
+    assert(2 + 2 == 4)
+  }
+}
+```
+
+```bash
+scala-cli test . --test-only 'tests.only*' 
+# tests.only.BarTests:
+#   + bar 0.045s
+```
+
+<!-- Expected:
+tests.only.BarTests:
++ bar
+-->
+
+## Filter test case 
+
+### Munit
+
+To run a specific test case inside the unit test suite pass `*exact-test-name*` as an argument to scala-cli:
+
+```scala title=BarTests.scala
+//> using lib "org.scalameta::munit::0.7.29"
+package tests.only
+
+class Tests extends munit.FunSuite {
+  test("bar") {
+    assert(2 + 2 == 5)
+  }
+  test("foo") {
+    assert(2 + 3 == 5)
+  }
+  test("foo-again") {
+    assert(2 + 3 == 5)
+  }
+}
+```
+```bash
+scala-cli test . --test-only 'tests.only*'  -- '*foo*'
+# tests.only.Tests:
+#   + foo 0.045s
+#   + foo-again 0.001s
+```
+
+<!-- Expected:
+tests.only.Tests:
++ foo
++ foo-again
+-->
+
+
 ## Test arguments
 
 You can pass test arguments to your test framework by passing them after `--`:

website/docs/reference/cli-options.md

@@ -1459,6 +1459,10 @@ Name of the test framework's runner class to use while running tests
 
 Fail if no test suites were run
 
+### `--test-only`
+
+Specify a glob pattern to filter the tests suite to be run.
+
 ## Uninstall options
 
 Available in commands:

website/docs/reference/scala-command/cli-options.md

@@ -1034,6 +1034,12 @@ Name of the test framework's runner class to use while running tests
 
 Fail if no test suites were run
 
+### `--test-only`
+
+`SHOULD have` per Scala Runner specification
+
+Specify a glob pattern to filter the tests suite to be run.
+
 ## Uninstall options
 
 Available in commands:

website/docs/reference/scala-command/runner-specification.md

@@ -3448,6 +3448,10 @@ Name of the test framework's runner class to use while running tests
 
 Fail if no test suites were run
 
+**--test-only**
+
+Specify a glob pattern to filter the tests suite to be run.
+
 <details><summary>
 
 ### Implementantation specific options