feat: Add regular expression support to text filters (#790)

Evan Light · claremacrae · elight · web-flow · commit e397477e9bb0 · 2022-08-12T09:29:05.000+01:00
* initial commit (blank message) * Integrating @claremacrae's fixes * Added PathField test courtesy of @claremacrae * Basic regex matching using PathField; need to support slash delimiters and regex params later * Regex now requires leading and trailing slashes; still no support for regex flags * test: Confirm regex matches is case-sensitive Co-Authored-By: Evan Light <10112+elight@users.noreply.github.com> * test: Add custom matcher for FilterOrErrorMessage Initially testing the PathField. Will be moved to a separate file later. Co-Authored-By: Evan Light <10112+elight@users.noreply.github.com> * refactor: Inline HeadingField.headingRegexp This is in preparation for creating it programmatically. * refactor: Make HeadingField.filterRegexp() programmatically * refactor: Move HeadingField.filterRegexp() to TextField * refactor: DescriptionField now uses TextField.filterRegexp() * test: Re-order expects for glance-ability * Handles regex flags (and linter won't STFU about it) * Refactor to DRY filtering * Test positive case with description field * Test negative regex match with description field * Linter fixes because I turned off my git hooks locally * Another 'fix' by the linter; I don't love its formatting here but I'll just go with it. * Add the link to the appropriate SO post for the regexp * Updated documentation * refactor: Fix '33:43 error Unnecessary escape character: \[ no-useless-escape' Authored-by: @claremacrae WebStorm says the behaviour is unchanged. * refactor: Fix 'Warning:(34, 75) Redundant character escape '\]' in RegExp' Authored-by: @claremacrae * refactor: Formatting changes made by 'yarn lint' Authored-by: @claremacrae * test: Remove console.log call from tests Authored-by: @claremacrae * fix: 'path regex does not match' works and is tested Authored-by: @claremacrae Co-authored-by: Clare Macrae <github@cfmacrae.fastmail.co.uk> Co-authored-by: Evan Light <10112+elight@users.noreply.github.com>
diff --git a/docs/queries/filters.md b/docs/queries/filters.md
@@ -134,6 +134,10 @@ As well as the date-related searches above, these filters search other propertie
 - `description (includes|does not include) <string>`
   - Matches case-insensitive (disregards capitalization).
   - Disregards the global filter when matching.
+- `description (regex matches|regex does not match) <JavaScript-style Regex>`
+  - Matches based on [JavaScript's RegExp implementation](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions)
+  - Supports [JavaScript RegExp Flags](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#advanced_searching_with_flags)
+  - **_Use with extreme care; this is a tool for software developers or people willing to spend a lot of time reading complicated documentation_**
 
 ### Priority
 
diff --git a/src/Query/Filter/DescriptionField.ts b/src/Query/Filter/DescriptionField.ts
@@ -9,17 +9,10 @@ import { TextField } from './TextField';
  * with the global filter (if any) removed.
  */
 export class DescriptionField extends TextField {
-    private static readonly descriptionRegexp =
-        /^description (includes|does not include) (.*)/;
-
     protected fieldName(): string {
         return 'description';
     }
 
-    protected filterRegexp(): RegExp {
-        return DescriptionField.descriptionRegexp;
-    }
-
     /**
      * Return the task's description, with any global tag removed
      * @param task
diff --git a/src/Query/Filter/HeadingField.ts b/src/Query/Filter/HeadingField.ts
@@ -5,13 +5,6 @@ import { TextField } from './TextField';
  *
  */
 export class HeadingField extends TextField {
-    private static readonly headingRegexp =
-        /^heading (includes|does not include) (.*)/;
-
-    protected filterRegexp(): RegExp {
-        return HeadingField.headingRegexp;
-    }
-
     protected fieldName(): string {
         return 'heading';
     }
diff --git a/src/Query/Filter/PathField.ts b/src/Query/Filter/PathField.ts
@@ -8,13 +8,6 @@ import { TextField } from './TextField';
  *
  */
 export class PathField extends TextField {
-    private static readonly pathRegexp =
-        /^path (includes|does not include) (.*)/;
-
-    protected filterRegexp(): RegExp {
-        return PathField.pathRegexp;
-    }
-
     protected fieldName(): string {
         return 'path';
     }
diff --git a/src/Query/Filter/TextField.ts b/src/Query/Filter/TextField.ts
@@ -8,23 +8,42 @@ import { FilterOrErrorMessage } from './Filter';
  * value, such as the description or file path.
  */
 export abstract class TextField extends Field {
+    private maybeNegate(match: boolean, filterMethod: String) {
+        return filterMethod.match(/not/) ? !match : match;
+    }
     public createFilterOrErrorMessage(line: string): FilterOrErrorMessage {
         const result = new FilterOrErrorMessage();
         const match = Field.getMatch(this.filterRegexp(), line);
         if (match !== null) {
             const filterMethod = match[1];
-            if (filterMethod === 'includes') {
-                result.filter = (task: Task) =>
-                    TextField.stringIncludesCaseInsensitive(
-                        this.value(task),
-                        match[2],
-                    );
-            } else if (match[1] === 'does not include') {
+            if (['includes', 'does not include'].includes(filterMethod)) {
                 result.filter = (task: Task) =>
-                    !TextField.stringIncludesCaseInsensitive(
-                        this.value(task),
-                        match[2],
+                    this.maybeNegate(
+                        TextField.stringIncludesCaseInsensitive(
+                            this.value(task),
+                            match[2],
+                        ),
+                        filterMethod,
                     );
+            } else if (
+                ['regex matches', 'regex does not match'].includes(filterMethod)
+            ) {
+                // Courtesy of https://stackoverflow.com/questions/17843691/javascript-regex-to-match-a-regex
+                const regexPattern =
+                    /\/((?![*+?])(?:[^\r\n[/\\]|\\.|\[(?:[^\r\n\]\\]|\\.)*])+)\/((?:g(?:im?|mi?)?|i(?:gm?|mg?)?|m(?:gi?|ig?)?)?)/;
+                const query = match[2].match(regexPattern);
+
+                if (query !== null) {
+                    result.filter = (task: Task) =>
+                        this.maybeNegate(
+                            this.value(task).match(
+                                new RegExp(query[1], query[2]),
+                            ) !== null,
+                            filterMethod,
+                        );
+                } else {
+                    result.error = `cannot parse regex (${this.fieldName()}); check your leading and trailing slashes for your query`;
+                }
             } else {
                 result.error = `do not understand query filter (${this.fieldName()})`;
             }
@@ -43,7 +62,11 @@ export abstract class TextField extends Field {
             .includes(needle.toLocaleLowerCase());
     }
 
-    protected abstract filterRegexp(): RegExp | null;
+    protected filterRegexp(): RegExp {
+        return new RegExp(
+            `^${this.fieldName()} (includes|does not include|regex matches|regex does not match) (.*)`,
+        );
+    }
 
     protected abstract fieldName(): string;
 
diff --git a/tests/Query/Filter/DescriptionField.test.ts b/tests/Query/Filter/DescriptionField.test.ts
@@ -15,6 +15,48 @@ function testDescriptionFilter(
     testTaskFilter(filter, task, expected);
 }
 
+declare global {
+    namespace jest {
+        interface Matchers<R> {
+            toMatchTaskWithDescription(description: string): R;
+        }
+
+        interface Expect {
+            toMatchTaskWithDescription(description: string): any;
+        }
+
+        interface InverseAsymmetricMatchers {
+            toMatchTaskWithDescription(description: string): any;
+        }
+    }
+}
+
+export function toMatchTaskWithDescription(
+    filter: FilterOrErrorMessage,
+    description: string,
+) {
+    const task = fromLine({
+        line: description,
+    });
+
+    const matches = filter.filter!(task);
+    if (!matches) {
+        return {
+            message: () => `unexpected failure to match task: ${description}`,
+            pass: false,
+        };
+    }
+
+    return {
+        message: () => `filter should not have matched task: ${description}`,
+        pass: true,
+    };
+}
+
+expect.extend({
+    toMatchTaskWithDescription,
+});
+
 describe('description', () => {
     it('ignores the global filter when filtering', () => {
         // Arrange
@@ -63,4 +105,34 @@ describe('description', () => {
         // Cleanup
         updateSettings(originalSettings);
     });
+
+    it('works with regex', () => {
+        // Arrange
+        const filter = new DescriptionField().createFilterOrErrorMessage(
+            'description regex matches /^task/',
+        );
+
+        // Assert
+        expect(filter).not.toMatchTaskWithDescription(
+            '- [ ] this does not start with the pattern',
+        );
+        expect(filter).toMatchTaskWithDescription(
+            '- [ ] task does start with the pattern',
+        );
+    });
+
+    it('works negating regexes', () => {
+        // Arrange
+        const filter = new DescriptionField().createFilterOrErrorMessage(
+            'description regex does not match /^task/',
+        );
+
+        // Assert
+        expect(filter).toMatchTaskWithDescription(
+            '- [ ] this does not start with the pattern',
+        );
+        expect(filter).not.toMatchTaskWithDescription(
+            '- [ ] task does start with the pattern',
+        );
+    });
 });
diff --git a/tests/Query/Filter/PathField.test.ts b/tests/Query/Filter/PathField.test.ts
@@ -12,6 +12,77 @@ function testTaskFilterForTaskWithPath(
     testFilter(filter, builder.path(path), expected);
 }
 
+declare global {
+    namespace jest {
+        interface Matchers<R> {
+            toMatchTaskWithPath(path: string): R;
+            toBeValid(): R;
+        }
+
+        interface Expect {
+            toMatchTaskWithPath(path: string): any;
+            toBeValid(): any;
+        }
+
+        interface InverseAsymmetricMatchers {
+            toMatchTaskWithPath(path: string): any;
+            toBeValid(): any;
+        }
+    }
+}
+
+export function toBeValid(filter: FilterOrErrorMessage) {
+    if (filter.filter === undefined) {
+        return {
+            message: () =>
+                'unexpected null filter: check your instruction matches your filter class',
+            pass: false,
+        };
+    }
+
+    if (filter.error !== undefined) {
+        return {
+            message: () =>
+                'unexpected error message in filter: check your instruction matches your filter class',
+            pass: false,
+        };
+    }
+
+    return {
+        message: () => 'filter is unexpectedly valid',
+        pass: true,
+    };
+}
+
+export function toMatchTaskWithPath(
+    filter: FilterOrErrorMessage,
+    path: string,
+) {
+    const builder = new TaskBuilder();
+    const task = builder.path(path).build();
+
+    const matches = filter.filter!(task);
+    if (!matches) {
+        return {
+            message: () => `unexpected failure to match task: ${path}`,
+            pass: false,
+        };
+    }
+
+    return {
+        message: () => `filter should not have matched task: ${path}`,
+        pass: true,
+    };
+}
+
+expect.extend({
+    toMatchTaskWithPath,
+});
+
+expect.extend({
+    toBeValid,
+});
+
 describe('path', () => {
     it('by path (includes)', () => {
         // Arrange
@@ -37,4 +108,49 @@ describe('path', () => {
         testTaskFilterForTaskWithPath(filter, '/some/path/file.md', false);
         testTaskFilterForTaskWithPath(filter, '/other/path/file.md', true);
     });
+
+    it('by path (regex matches)', () => {
+        // Arrange
+        const filter = new PathField().createFilterOrErrorMessage(
+            'path regex matches /w.bble/',
+        );
+
+        // Assert
+        expect(filter).toBeValid();
+        expect(filter).toMatchTaskWithPath('/some/path/wibble.md');
+        expect(filter).toMatchTaskWithPath('/some/path/wobble.md');
+        expect(filter).not.toMatchTaskWithPath('');
+        expect(filter).not.toMatchTaskWithPath('/some/path/WobblE.md'); // confirm case-sensitive
+        expect(filter).not.toMatchTaskWithPath('/other/path/file.md');
+    });
+
+    it('by path (regex matches) with flags', () => {
+        // Arrange
+        const filter = new PathField().createFilterOrErrorMessage(
+            'path regex matches /w.bble/i',
+        );
+
+        // Assert
+        expect(filter).toBeValid();
+        expect(filter).toMatchTaskWithPath('/some/path/wibble.md');
+        expect(filter).toMatchTaskWithPath('/some/path/wobble.md');
+        expect(filter).not.toMatchTaskWithPath('');
+        expect(filter).toMatchTaskWithPath('/some/path/WobblE.md'); // confirm case-insensitive (flag)
+        expect(filter).not.toMatchTaskWithPath('/other/path/file.md');
+    });
+
+    it('by path (regex does not match)', () => {
+        // Arrange
+        const filter = new PathField().createFilterOrErrorMessage(
+            'path regex does not match /w.bble/',
+        );
+
+        // Assert
+        expect(filter).toBeValid();
+        expect(filter).not.toMatchTaskWithPath('/some/path/wibble.md');
+        expect(filter).not.toMatchTaskWithPath('/some/path/wobble.md');
+        expect(filter).toMatchTaskWithPath('');
+        expect(filter).toMatchTaskWithPath('/some/path/WobblE.md'); // confirm case-sensitive
+        expect(filter).toMatchTaskWithPath('/other/path/file.md');
+    });
 });

Original file line number	Diff line number	Diff line change
`@@ -5,13 +5,6 @@ import { TextField } from './TextField';`
`5`	`5`	`*`
`6`	`6`	`*/`
`7`	`7`	`export class HeadingField extends TextField {`
`8`		`- private static readonly headingRegexp =`
`9`		`- /^heading (includes\|does not include) (.*)/;`
`10`		`-`
`11`		`- protected filterRegexp(): RegExp {`
`12`		`- return HeadingField.headingRegexp;`
`13`		`- }`
`14`		`-`
`15`	`8`	`protected fieldName(): string {`
`16`	`9`	`return 'heading';`
`17`	`10`	`}`
Original file line number	Diff line number	Diff line change
`@@ -8,13 +8,6 @@ import { TextField } from './TextField';`
`8`	`8`	`*`
`9`	`9`	`*/`
`10`	`10`	`export class PathField extends TextField {`
`11`		`- private static readonly pathRegexp =`
`12`		`- /^path (includes\|does not include) (.*)/;`
`13`		`-`
`14`		`- protected filterRegexp(): RegExp {`
`15`		`- return PathField.pathRegexp;`
`16`		`- }`
`17`		`-`
`18`	`11`	`protected fieldName(): string {`
`19`	`12`	`return 'path';`
`20`	`13`	`}`