feat: add option to omit all filtered tests (#84)

* add failing test for omitted tests

* feat: add variable to omit filtered tests

Author: bahmutov

.github/workflows/ci.yml

@@ -185,6 +185,15 @@ jobs:
             --env grep="works 2" \
             --expect-exactly expects/specify.json
 
+      # instead of making filtered tests pending
+      # with it.skip, completely omit them
+      - name: Omit filtered tests 🧪
+        run: |
+          npx cypress-expect \
+            --spec cypress/integration/spec.js \
+            --env grep="works 2",grepOmitFiltered=true \
+            --expect-exactly expects/omit-filtered.json
+
   tests3:
     runs-on: ubuntu-20.04
     needs: install

README.md

@@ -217,6 +217,36 @@ Note 3: if there are no files remaining after filtering, the plugin prints a war
 }
 ```
 
+### omit filtered tests
+
+By default, all filtered tests are made _pending_ using `it.skip` method. If you want to completely omit them, pass the environment variable `grepOmitFiltered=true`.
+
+Pending filtered tests
+
+```
+cypress run --env grep="works 2"
+```
+
+![Pending tests](./images/includes-pending.png)
+
+Omit filtered tests
+
+```
+cypress run --env grep="works 2",grepOmitFiltered=true
+```
+
+![Only running tests remaining](./images/omit-pending.png)
+
+**Tip:** you can set this environment variable in the `cypress.json` file to enable it by default and skip using the environment variable:
+
+```json
+{
+  "env": {
+    "grepOmitFiltered": true
+  }
+}
+```
+
 ### grep untagged tests
 
 Sometimes you want to run only the tests without any tags, and these tests are inside the describe blocks without any tags.

expects/omit-filtered.json

@@ -0,0 +1,4 @@
+{
+  "works 2 @tag1": "passed",
+  "works 2 @tag1 @tag2": "passed"
+}

images/includes-pending.png

@@ -35,6 +35,12 @@ function cypressGrepPlugin(config) {
     console.log('cypress-grep: running untagged tests')
   }
 
+  const omitFiltered =
+    config.env.grepOmitFiltered || config.env['grep-omit-filtered']
+  if (omitFiltered) {
+    console.log('cypress-grep: will omit filtered tests')
+  }
+
   const grepFilterSpecs = config.env.grepFilterSpecs === true
   if (grepFilterSpecs) {
     if (grep) {

images/omit-pending.png

@@ -42,7 +42,11 @@ function cypressGrep() {
     Cypress.env('burn') ||
     1
 
-  debug('grep %o', { grep, grepTags, grepBurn })
+  /** @type {boolean} Omit filtered tests completely */
+  const omitFiltered =
+    Cypress.env('grepOmitFiltered') || Cypress.env('grep-omit-filtered')
+
+  debug('grep %o', { grep, grepTags, grepBurn, omitFiltered })
   if (!Cypress._.isInteger(grepBurn) || grepBurn < 1) {
     throw new Error(`Invalid grep burn value: ${grepBurn}`)
   }
@@ -112,8 +116,13 @@ function cypressGrep() {
       return _it(name, options, callback)
     }
 
-    // skip tests without grep string in their names
-    return _it.skip(name, options, callback)
+    if (omitFiltered) {
+      // omit the filtered tests completely
+      return
+    } else {
+      // skip tests without grep string in their names
+      return _it.skip(name, options, callback)
+    }
   }
 
   // list of "describe" suites for the current test