Add --filter-not setting (#59)

Author: shabalind

README.md

@@ -27,18 +27,26 @@ Example:
 ```terminal
 $ swift run -c release BenchmarkMinimalExample -h
 [3/3] Linking BenchmarkMinimalExample
-USAGE: benchmark-command [--allow-debug-build] [--filter <filter>] [--iterations <iterations>] [--warmup-iterations <warmup-iterations>] [--min-time <min-time>] [--max-iterations <max-iterations>]
+USAGE: benchmark-command [--allow-debug-build] [--filter <filter>] [--filter-not <filter-not>] [--iterations <iterations>] [--warmup-iterations <warmup-iterations>] [--min-time <min-time>] [--max-iterations <max-iterations>] [--time-unit <time-unit>] [--inverse-time-unit <inverse-time-unit>] [--columns <columns>] [--format <format>]
 
 OPTIONS:
   --allow-debug-build     Overrides check to verify optimized build.
   --filter <filter>       Run only benchmarks whose names match the regular expression.
+  --filter-not <filter-not>
+                          Exclude benchmarks whose names match the regular expression.
   --iterations <iterations>
                           Number of iterations to run.
   --warmup-iterations <warmup-iterations>
                           Number of warm-up iterations to run.
   --min-time <min-time>   Minimal time to run when automatically detecting number iterations.
   --max-iterations <max-iterations>
-                          Maximum number of iterations to run when automatically detecting number iterations.
+                          Maximum number of iterations to run when automatically detecting number
+                          iterations.
+  --time-unit <time-unit> Time unit used to report the timing results.
+  --inverse-time-unit <inverse-time-unit>
+                          Inverse time unit used to report throughput results.
+  --columns <columns>     Comma-separated list of column names to show.
+  --format <format>       Output format (valid values are: json, csv, console, none).
   -h, --help              Show help information.
 
 $ swift run -c release BenchmarkMinimalExample

Sources/Benchmark/BenchmarkArguments.swift

@@ -23,6 +23,9 @@ public struct BenchmarkArguments: ParsableArguments {
     @Option(help: "Run only benchmarks whose names match the regular expression.")
     var filter: String?
 
+    @Option(help: "Exclude benchmarks whose names match the regular expression.")
+    var filterNot: String?
+
     @Option(help: "Number of iterations to run.")
     var iterations: Int?
 
@@ -57,6 +60,9 @@ public struct BenchmarkArguments: ParsableArguments {
         if let value = filter {
             result.append(Filter(value))
         }
+        if let value = filterNot {
+            result.append(FilterNot(value))
+        }
         if let value = iterations {
             result.append(Iterations(value))
         }

Sources/Benchmark/BenchmarkFilter.swift

@@ -15,9 +15,11 @@
 import Foundation
 
 internal struct BenchmarkFilter {
+    let negate: Bool
     let underlying: NSRegularExpression?
 
-    init(_ regularExpression: String?) throws {
+    init(_ regularExpression: String?, negate: Bool) throws {
+        self.negate = negate
         if regularExpression != nil {
             self.underlying = try NSRegularExpression(
                 pattern: regularExpression!,
@@ -33,7 +35,9 @@ internal struct BenchmarkFilter {
         } else {
             let str = "\(suiteName).\(benchmarkName)"
             let range = NSRange(location: 0, length: str.utf16.count)
-            return underlying!.firstMatch(in: str, range: range) != nil
+            var result = underlying!.firstMatch(in: str, range: range) != nil
+            result = negate ? !result : result
+            return result
         }
     }
 }

Sources/Benchmark/BenchmarkRunner.swift

@@ -74,11 +74,16 @@ public struct BenchmarkRunner {
             self.settings,
         ])
 
-        let filter = try BenchmarkFilter(settings.filter)
+        let filter = try BenchmarkFilter(settings.filter, negate: false)
         if !filter.matches(suiteName: suite.name, benchmarkName: benchmark.name) {
             return
         }
 
+        let filterNot = try BenchmarkFilter(settings.filterNot, negate: true)
+        if !filterNot.matches(suiteName: suite.name, benchmarkName: benchmark.name) {
+            return
+        }
+
         progress.report(running: benchmark.name, suite: suite.name)
         let totalStart = now()
 

Sources/Benchmark/BenchmarkSetting.swift

@@ -54,6 +54,14 @@ public struct Filter: BenchmarkSetting {
     }
 }
 
+/// A regex string used to exclude benchmarks that should be run.
+public struct FilterNot: BenchmarkSetting {
+    public var value: String
+    public init(_ value: String) {
+        self.value = value
+    }
+}
+
 /// A minimal (total) time that iterations has to run
 /// to be considered significant.
 public struct MinTime: BenchmarkSetting {
@@ -184,6 +192,11 @@ public struct BenchmarkSettings {
         return self[Filter.self]?.value
     }
 
+    /// Convenience accessor for the Filter setting.
+    public var filterNot: String? {
+        return self[FilterNot.self]?.value
+    }
+
     /// Convenience accessor for the MinTime setting.
     public var minTime: Double {
         if let value = self[MinTime.self]?.value {

Tests/BenchmarkTests/BenchmarkColumnTests.swift

@@ -55,7 +55,7 @@ final class BenchmarkColumnTests: XCTestCase {
 
     func testRenamed() {
         let time = BenchmarkColumn.registry["time"]!
-        XCTAssertEqual(time.name, "time") 
+        XCTAssertEqual(time.name, "time")
         let mytime = time.renamed("mytime")
         XCTAssertEqual(mytime.name, "mytime")
     }

Tests/BenchmarkTests/BenchmarkCommandTests.swift

@@ -42,19 +42,34 @@ final class BenchmarkCommandTests: XCTestCase {
 
     func testParseFilter() throws {
         AssertParse(["--filter", "bar", "--allow-debug-build"]) { settings in
-            let filter = try! BenchmarkFilter(settings.filter)
+            let filter = try! BenchmarkFilter(settings.filter, negate: false)
             XCTAssertFalse(filter.matches(suiteName: "foo", benchmarkName: "baz"))
             XCTAssert(filter.matches(suiteName: "foo", benchmarkName: "bar"))
         }
 
         AssertParse(["--filter", "foo.bar", "--allow-debug-build"]) { settings in
-            let filter = try! BenchmarkFilter(settings.filter)
+            let filter = try! BenchmarkFilter(settings.filter, negate: false)
             XCTAssertFalse(filter.matches(suiteName: "foo", benchmarkName: "baz"))
             XCTAssertFalse(filter.matches(suiteName: "foobar", benchmarkName: "baz"))
             XCTAssert(filter.matches(suiteName: "foo", benchmarkName: "bar"))
         }
     }
 
+    func testParseFilterNot() throws {
+        AssertParse(["--filter-not", "bar", "--allow-debug-build"]) { settings in
+            let filter = try! BenchmarkFilter(settings.filterNot, negate: true)
+            XCTAssert(filter.matches(suiteName: "foo", benchmarkName: "baz"))
+            XCTAssertFalse(filter.matches(suiteName: "foo", benchmarkName: "bar"))
+        }
+
+        AssertParse(["--filter-not", "foo.bar", "--allow-debug-build"]) { settings in
+            let filter = try! BenchmarkFilter(settings.filterNot, negate: true)
+            XCTAssert(filter.matches(suiteName: "foo", benchmarkName: "baz"))
+            XCTAssert(filter.matches(suiteName: "foobar", benchmarkName: "baz"))
+            XCTAssertFalse(filter.matches(suiteName: "foo", benchmarkName: "bar"))
+        }
+    }
+
     func testParseIterations() throws {
         AssertParse(["--iterations", "42", "--allow-debug-build"]) { settings in
             XCTAssertEqual(settings.iterations, 42)
@@ -113,6 +128,7 @@ final class BenchmarkCommandTests: XCTestCase {
         ("testAllowDebugBuild", testAllowDebugBuild),
         ("testDebugBuildError", testDebugBuildError),
         ("testParseFilter", testParseFilter),
+        ("testParseFilterNot", testParseFilterNot),
         ("testParseIterations", testParseIterations),
         ("testParseZeroIterations", testParseZeroIterations),
         ("testParseWarmupIterations", testParseWarmupIterations),