BAL-3498: enhance rule-engine evaluation logic with path comparison (#3069)

* feat(rule-engine): enhance rule evaluation logic with path comparison

- Introduce isPathComparison flag in Rule schema for clearer distinction
- Update extractValue method to handle operators with and without path comparison
- Adjust validation logic to accommodate new extracted value structure

* fix(rule-engine): correct path comparison validation logic

- Ensure 'isPathComparison' is only true when present in the rule
- Update condition to prevent false positives in comparison checks

* refactor(rule-engine): improve operator extraction logic and constants usage

- Move OPERATORS_WITHOUT_PATH_COMPARISON to constants for better reusability
- Refactor extraction logic to use isObject utility for clarity
- Update imports to reflect the new constants structure

* fix(rule-engine): improve rule extraction validation

- Refactor validation logic for extracted values
- Utilize 'in' operator for better readability and accuracy

* feat(tests): enhance rule engine tests with path comparison functionality

- Add isPathComparison flag to various rule definitions
- Implement new unit tests for path comparison scenarios
- Update integration tests to include path comparison in validation

Author: shanegrouber

packages/common/src/rule-engine/operators/constants.ts

@@ -31,3 +31,9 @@ export const OperationHelpers = {
   [OPERATION.NOT_IN]: NOT_IN,
   [OPERATION.AML_CHECK]: AML_CHECK,
 } as const;
+
+export const OPERATORS_WITHOUT_PATH_COMPARISON = [
+  OPERATION.AML_CHECK,
+  OPERATION.BETWEEN,
+  OPERATION.LAST_YEAR,
+] as const;

packages/common/src/rule-engine/operators/helpers.ts

@@ -14,7 +14,7 @@ import { z, ZodSchema } from 'zod';
 import { BetweenSchema, LastYearsSchema, PrimitiveArraySchema, PrimitiveSchema } from './schemas';
 
 import { ValidationFailedError, DataValueNotFoundError } from '../errors';
-import { OperationHelpers } from './constants';
+import { OperationHelpers, OPERATORS_WITHOUT_PATH_COMPARISON } from './constants';
 import { Rule } from '@/rule-engine';
 import { EndUserAmlHitsSchema } from '@/schemas';
 
@@ -40,11 +40,30 @@ export abstract class BaseOperator<TDataValue = Primitive, TConditionValue = Pri
   extractValue(data: unknown, rule: Rule) {
     const value = get(data, rule.key);
 
-    if (value === undefined || value === null) {
-      throw new DataValueNotFoundError(rule.key);
+    const isPathComparison =
+      !OPERATORS_WITHOUT_PATH_COMPARISON.includes(
+        rule.operator as (typeof OPERATORS_WITHOUT_PATH_COMPARISON)[number],
+      ) &&
+      'isPathComparison' in rule &&
+      rule.isPathComparison;
+
+    if (!isPathComparison) {
+      if (value === undefined || value === null) {
+        throw new DataValueNotFoundError(rule.key);
+      }
+
+      return value;
+    }
+
+    const comparisonValueAsPath = rule.value as string;
+
+    const evaluatedComparisonValue = get(data, comparisonValueAsPath);
+
+    if (evaluatedComparisonValue === undefined || evaluatedComparisonValue === null) {
+      throw new DataValueNotFoundError(comparisonValueAsPath);
     }
 
-    return value;
+    return { value, comparisonValue: evaluatedComparisonValue };
   }
 
   execute(dataValue: TDataValue, conditionValue: TConditionValue) {

packages/common/src/rule-engine/rules/schemas.ts

@@ -39,11 +39,13 @@ export const RuleSchema = z.discriminatedUnion('operator', [
     key: z.string(),
     operator: z.literal(OPERATION.EQUALS),
     value: PrimitiveSchema,
+    isPathComparison: z.boolean().default(false),
   }),
   z.object({
     key: z.string(),
     operator: z.literal(OPERATION.NOT_EQUALS),
     value: PrimitiveSchema,
+    isPathComparison: z.boolean().default(false),
   }),
   z.object({
     key: z.string(),
@@ -54,21 +56,25 @@ export const RuleSchema = z.discriminatedUnion('operator', [
     key: z.string(),
     operator: z.literal(OPERATION.GT),
     value: PrimitiveSchema,
+    isPathComparison: z.boolean().default(false),
   }),
   z.object({
     key: z.string(),
     operator: z.literal(OPERATION.LT),
     value: PrimitiveSchema,
+    isPathComparison: z.boolean().default(false),
   }),
   z.object({
     key: z.string(),
     operator: z.literal(OPERATION.GTE),
     value: PrimitiveSchema,
+    isPathComparison: z.boolean().default(false),
   }),
   z.object({
     key: z.string(),
     operator: z.literal(OPERATION.LTE),
     value: PrimitiveSchema,
+    isPathComparison: z.boolean().default(false),
   }),
   z.object({
     key: z.string(),
@@ -79,16 +85,19 @@ export const RuleSchema = z.discriminatedUnion('operator', [
     key: z.string(),
     operator: z.literal(OPERATION.IN),
     value: PrimitiveArraySchema,
+    isPathComparison: z.boolean().default(false),
   }),
   z.object({
     key: z.string(),
     operator: z.literal(OPERATION.IN_CASE_INSENSITIVE),
     value: PrimitiveArraySchema,
+    isPathComparison: z.boolean().default(false),
   }),
   z.object({
     key: z.string(),
     operator: z.literal(OPERATION.NOT_IN),
     value: PrimitiveArraySchema,
+    isPathComparison: z.boolean().default(false),
   }),
 ]);
 

services/workflows-service/prisma/data-migrations

@@ -8,6 +8,7 @@ import {
   OPERATOR,
   RuleSchema,
   ValidationFailedError,
+  isObject,
 } from '@ballerine/common';
 
 export const validateRule = (rule: Rule, data: any): RuleResult => {
@@ -23,11 +24,17 @@ export const validateRule = (rule: Rule, data: any): RuleResult => {
     throw new OperatorNotFoundError(rule.operator);
   }
 
-  const value = operator.extractValue(data, rule);
+  const extractedValue = operator.extractValue(data, rule);
+
+  const isPathComparison =
+    isObject(extractedValue) && 'value' in extractedValue && 'comparisonValue' in extractedValue;
+
+  const { value, comparisonValue } = isPathComparison
+    ? extractedValue
+    : { value: extractedValue, comparisonValue: rule.value };
 
   try {
-    // @ts-expect-error - rule
-    const result = operator.execute(value, rule.value);
+    const result = operator.execute(value, comparisonValue);
 
     return { status: result ? 'PASSED' : 'FAILED', error: undefined };
   } catch (error) {

services/workflows-service/src/rule-engine/core/rule-engine.ts

@@ -26,6 +26,7 @@ describe('Rule Engine', () => {
           key: 'country',
           operator: OPERATION.EQUALS,
           value: 'US',
+          isPathComparison: false,
         },
         {
           operator: OPERATOR.AND,
@@ -34,6 +35,7 @@ describe('Rule Engine', () => {
               key: 'name',
               operator: OPERATION.EQUALS,
               value: 'John',
+              isPathComparison: false,
             },
             {
               operator: OPERATOR.OR,
@@ -42,11 +44,13 @@ describe('Rule Engine', () => {
                   key: 'age',
                   operator: OPERATION.GT,
                   value: 40,
+                  isPathComparison: false,
                 },
                 {
                   key: 'age',
                   operator: OPERATION.LTE,
                   value: 35,
+                  isPathComparison: false,
                 },
               ],
             },
@@ -73,6 +77,7 @@ describe('Rule Engine', () => {
           key: 'nonexistent',
           operator: OPERATION.EQUALS,
           value: 'US',
+          isPathComparison: false,
         },
       ],
     };
@@ -95,6 +100,7 @@ describe('Rule Engine', () => {
           operator: 'UNKNOWN',
           // @ts-ignore - intentionally using an unknown operator
           value: 'US',
+          isPathComparison: false,
         },
       ],
     };
@@ -141,6 +147,7 @@ describe('Rule Engine', () => {
           key: 'country',
           operator: OPERATION.EQUALS,
           value: 'CA',
+          isPathComparison: false,
         },
       ],
     };
@@ -220,6 +227,7 @@ describe('Rule Engine', () => {
           key: '',
           operator: OPERATION.EQUALS,
           value: 'US',
+          isPathComparison: false,
         },
       ],
     };
@@ -232,6 +240,7 @@ describe('Rule Engine', () => {
         "error": [DataValueNotFoundError: Field  is missing or null],
         "message": "Field  is missing or null",
         "rule": {
+          "isPathComparison": false,
           "key": "",
           "operator": "EQUALS",
           "value": "US",
@@ -493,6 +502,7 @@ describe('Rule Engine', () => {
             key: 'pluginsOutput.companySanctions.data.length',
             operator: OPERATION.NOT_EQUALS,
             value: 0,
+            isPathComparison: false,
           },
         ],
       };
@@ -506,6 +516,7 @@ describe('Rule Engine', () => {
         {
           "error": undefined,
           "rule": {
+            "isPathComparison": false,
             "key": "pluginsOutput.companySanctions.data.length",
             "operator": "NOT_EQUALS",
             "value": 0,
@@ -526,6 +537,7 @@ describe('Rule Engine', () => {
         {
           "error": undefined,
           "rule": {
+            "isPathComparison": false,
             "key": "pluginsOutput.companySanctions.data.length",
             "operator": "NOT_EQUALS",
             "value": 0,
@@ -545,6 +557,7 @@ describe('Rule Engine', () => {
             key: 'entity.data.country',
             operator: OPERATION.IN,
             value: ['IL', 'AF', 'US', 'GB'],
+            isPathComparison: false,
           },
         ],
       };
@@ -558,6 +571,7 @@ describe('Rule Engine', () => {
         {
           "error": undefined,
           "rule": {
+            "isPathComparison": false,
             "key": "entity.data.country",
             "operator": "IN",
             "value": [
@@ -583,6 +597,7 @@ describe('Rule Engine', () => {
         {
           "error": undefined,
           "rule": {
+            "isPathComparison": false,
             "key": "entity.data.country",
             "operator": "IN",
             "value": [
@@ -607,6 +622,7 @@ describe('Rule Engine', () => {
             key: 'country',
             operator: OPERATION.IN_CASE_INSENSITIVE,
             value: ['us', 'ca'],
+            isPathComparison: false,
           },
         ],
       };
@@ -633,6 +649,7 @@ describe('Rule Engine', () => {
             key: 'countries',
             operator: OPERATION.IN_CASE_INSENSITIVE,
             value: ['us', 'ca'],
+            isPathComparison: false,
           },
         ],
       };
@@ -661,6 +678,7 @@ describe('Rule Engine', () => {
             key: 'entity.data.country',
             operator: OPERATION.NOT_IN,
             value: ['IL', 'CA', 'US', 'GB'],
+            isPathComparison: false,
           },
         ],
       };
@@ -674,6 +692,7 @@ describe('Rule Engine', () => {
         {
           "error": undefined,
           "rule": {
+            "isPathComparison": false,
             "key": "entity.data.country",
             "operator": "NOT_IN",
             "value": [
@@ -699,6 +718,7 @@ describe('Rule Engine', () => {
         {
           "error": undefined,
           "rule": {
+            "isPathComparison": false,
             "key": "entity.data.country",
             "operator": "NOT_IN",
             "value": [
@@ -1038,4 +1058,69 @@ describe('Rule Engine', () => {
       `);
     });
   });
+
+  describe('Path comparison', () => {
+    it('should compare values from two different paths', () => {
+      const ruleSetExample: RuleSet = {
+        operator: OPERATOR.AND,
+        rules: [
+          {
+            key: 'pluginsOutput.businessInformation.data[0].companyName',
+            operator: OPERATION.NOT_EQUALS,
+            value: 'entity.data.companyName',
+            isPathComparison: true,
+          },
+        ],
+      };
+
+      const engine = RuleEngine(ruleSetExample);
+      const result = engine.run(context);
+      expect(result).toBeDefined();
+      expect(result).toHaveLength(1);
+      expect(result[0]).toMatchInlineSnapshot(`
+        {
+          "error": undefined,
+          "rule": {
+            "isPathComparison": true,
+            "key": "pluginsOutput.businessInformation.data[0].companyName",
+            "operator": "NOT_EQUALS",
+            "value": "entity.data.companyName",
+          },
+          "status": "PASSED",
+        }
+      `);
+    });
+
+    it('should handle invalid paths', () => {
+      const ruleSetExample: RuleSet = {
+        operator: OPERATOR.AND,
+        rules: [
+          {
+            key: 'pluginsOutput.businessInformation.data[0].companyName',
+            operator: OPERATION.NOT_EQUALS,
+            value: 'entity.invalid.path',
+            isPathComparison: true,
+          },
+        ],
+      };
+
+      const engine = RuleEngine(ruleSetExample);
+      const result = engine.run(context);
+      expect(result).toBeDefined();
+      expect(result).toHaveLength(1);
+      expect(result[0]).toMatchInlineSnapshot(`
+        {
+          "error": [DataValueNotFoundError: Field entity.invalid.path is missing or null],
+          "message": "Field entity.invalid.path is missing or null",
+          "rule": {
+            "isPathComparison": true,
+            "key": "pluginsOutput.businessInformation.data[0].companyName",
+            "operator": "NOT_EQUALS",
+            "value": "entity.invalid.path",
+          },
+          "status": "FAILED",
+        }
+      `);
+    });
+  });
 });

services/workflows-service/src/rule-engine/core/test/rule-engine.unit.test.ts

@@ -21,11 +21,13 @@ describe('RuleEngineService', () => {
           key: 'single',
           operator: 'IN_CASE_INSENSITIVE',
           value: ['sole'],
+          isPathComparison: false,
         },
         {
           key: 'array',
           operator: 'IN_CASE_INSENSITIVE',
           value: ['ownership'],
+          isPathComparison: false,
         },
       ],
     };