feat: Added DataProcessors to handle mutations depending on the StorageEngine

Marco Franceschi · Marco Franceschi · commit e061c80cd55a · 2022-03-16T15:44:38.000-03:00
diff --git a/src/plugins/policyPack/index.ts b/src/plugins/policyPack/index.ts
@@ -13,6 +13,8 @@ import {
 import RulesEngine from '../../rules-engine'
 import { Result, Rule, Severity } from '../../rules-engine/types'
 import Plugin, { ConfiguredPlugin, PluginManager } from '../types'
+import DgraphDataProcessor from '../../rules-engine/data-processors/dgraph-data-processor'
+import DataProcessor from '../../rules-engine/data-processors/data-processor'
 
 export default class PolicyPackPlugin extends Plugin {
   constructor({
@@ -61,6 +63,10 @@ export default class PolicyPackPlugin extends Plugin {
     }
   } = {}
 
+  private dataProcessors: {
+    [name: string]: DataProcessor
+  } = {}
+
   private async getPolicyPackPackage({
     policyPack,
     pluginManager,
@@ -179,6 +185,24 @@ export default class PolicyPackPlugin extends Plugin {
     return findings
   }
 
+  // TODO: Generalize data processor moving storage module to SDK with its interfaces
+  private getDataProcessor({
+    entity,
+    provider,
+  }: {
+    entity: string
+    provider: string
+  }): DataProcessor {
+    const dataProcessorKey = `${provider}${entity}`
+    if (this.dataProcessors[dataProcessorKey]) {
+      return this.dataProcessors[dataProcessorKey]
+    }
+
+    const dataProcessor = new DgraphDataProcessor(provider, entity)
+    this.dataProcessors[dataProcessorKey] = dataProcessor
+    return dataProcessor
+  }
+
   async configure(
     pluginManager: PluginManager,
     plugins: ConfiguredPlugin[]
@@ -263,8 +287,6 @@ export default class PolicyPackPlugin extends Plugin {
       schemaMap?: SchemaMap
     }) => void
   }): Promise<any> {
-    console.log('storage type', typeof storageEngine, storageEngine)
-
     !isEmpty(this.policyPacksPlugins) &&
       this.logger.info(
         `Beginning ${chalk.italic.green('RULES')} for ${this.provider.name}`
@@ -296,8 +318,14 @@ export default class PolicyPackPlugin extends Plugin {
           storageEngine,
         })
 
+        // Data Processor
+        const dataProcessor = this.getDataProcessor({
+          entity,
+          provider: this.provider.name,
+        })
+
         // Prepare mutations
-        const mutations = engine?.prepareMutations(findings)
+        const mutations = dataProcessor.prepareMutations(findings)
 
         // Save connections
         processConnectionsBetweenEntities({
diff --git a/src/rules-engine/data-processors/data-processor.ts b/src/rules-engine/data-processors/data-processor.ts
@@ -0,0 +1,11 @@
+import { Entity } from '../../types'
+import { RuleFinding } from '../types'
+
+export default interface DataProcessor {
+  /**
+   * Transforms RuleFinding array into a mutation array for GraphQL
+   * @param findings resulted findings during rules execution
+   * @returns {Entity[]} Array of generated mutations
+   */
+  prepareMutations: (findings: RuleFinding[]) => Entity[]
+}
diff --git a/src/rules-engine/data-processors/dgraph-data-processor.ts b/src/rules-engine/data-processors/dgraph-data-processor.ts
@@ -0,0 +1,146 @@
+import groupBy from 'lodash/groupBy'
+import isEmpty from 'lodash/isEmpty'
+import { Entity } from '../../types'
+import { RuleFinding } from '../types'
+import DataProcessor from './data-processor'
+
+export default class DgraphDataProcessor implements DataProcessor {
+  private readonly providerName
+
+  private readonly entityName
+
+  constructor(providerName: string, entityName: string) {
+    this.providerName = providerName
+    this.entityName = entityName
+  }
+
+  /**
+   * Prepare the mutations for overall provider findings
+   * @param findings RuleFinding array
+   * @returns A formatted Entity array
+   */
+  private prepareProviderMutations = (
+    findings: RuleFinding[] = []
+  ): Entity[] => {
+    // Prepare provider schema connections
+    return [
+      {
+        name: `${this.providerName}Findings`,
+        mutation: `
+        mutation($input: [Add${this.providerName}FindingsInput!]!) {
+          add${this.providerName}Findings(input: $input, upsert: true) {
+            numUids
+          }
+        }
+        `,
+        data: {
+          id: `${this.providerName}-provider`,
+        },
+      },
+      {
+        name: `${this.providerName}Findings`,
+        mutation: `mutation update${this.providerName}Findings($input: Update${this.providerName}FindingsInput!) {
+            update${this.providerName}Findings(input: $input) {
+              numUids
+            }
+          }
+          `,
+        data: {
+          filter: {
+            id: { eq: `${this.providerName}-provider` },
+          },
+          set: {
+            [`${this.entityName}Findings`]: findings.map(
+              ({ typename, ...rest }) => ({ ...rest })
+            ),
+          },
+        },
+      },
+    ]
+  }
+
+  private prepareProcessedMutations(findingsByType: {
+    [resource: string]: RuleFinding[]
+  }): Entity[] {
+    const mutations = []
+
+    for (const findingType in findingsByType) {
+      if (!isEmpty(findingType)) {
+        // Group Findings by resource
+        const findingsByResource = groupBy(
+          findingsByType[findingType],
+          'resourceId'
+        )
+
+        for (const resource in findingsByResource) {
+          if (resource) {
+            const data = (
+              (findingsByResource[resource] as RuleFinding[]) || []
+            ).map(({ typename, ...properties }) => properties)
+
+            // Create dynamically update mutations by resource
+            const updateMutation = {
+              name: `${this.providerName}${this.entityName}Findings`,
+              mutation: `mutation update${findingType}($input: Update${findingType}Input!) {
+                update${findingType}(input: $input) {
+                  numUids
+                }
+              }
+              `,
+              data: {
+                filter: {
+                  id: { eq: resource },
+                },
+                set: {
+                  [`${this.entityName}Findings`]: data,
+                },
+              },
+            }
+
+            mutations.push(updateMutation)
+          }
+        }
+      }
+    }
+
+    return mutations
+  }
+
+  private prepareManualMutations(findings: RuleFinding[] = []): Entity[] {
+    const manualFindings = findings.map(({ typename, ...finding }) => ({
+      ...finding,
+    }))
+    return manualFindings.length > 0
+      ? [
+          {
+            name: `${this.providerName}${this.entityName}Findings`,
+            mutation: `
+      mutation($input: [Add${this.providerName}${this.entityName}FindingsInput!]!) {
+        add${this.providerName}${this.entityName}Findings(input: $input, upsert: true) {
+          numUids
+        }
+      }
+      `,
+            data: manualFindings,
+          },
+        ]
+      : []
+  }
+
+  // TODO: Optimize generated mutations number
+  prepareMutations = (findings: RuleFinding[] = []): Entity[] => {
+    // Group Findings by schema type
+    const { manual, ...processedRules } = groupBy(findings, 'typename')
+
+    // Prepare processed rules mutations
+    const processedRulesData = this.prepareProcessedMutations(processedRules)
+
+    // Prepare manual mutations
+    const manualRulesData = this.prepareManualMutations(manual)
+
+    // Prepare provider mutations
+    const providerData = this.prepareProviderMutations(findings)
+
+    return [...manualRulesData, ...processedRulesData, ...providerData]
+  }
+}
diff --git a/src/rules-engine/evaluators/json-evaluator.ts b/src/rules-engine/evaluators/json-evaluator.ts
@@ -78,13 +78,6 @@ export default class JsonEvaluator implements RuleEvaluator<JsonRule> {
       }
       return true
     },
-    compare: async (_, conditions: Condition[], data) => {
-      for (let i = 0; i < conditions.length; i++) {
-        // if 1 is false, it's false
-        if (!(await this.evaluateCondition(conditions[i], data))) return false
-      }
-      return true
-    },
     array_all: async (array = [], conditions: Condition, data) => {
       // an AND, but with every resource item
       const baseElementPath = data.elementPath
diff --git a/src/rules-engine/evaluators/manual-evaluator.ts b/src/rules-engine/evaluators/manual-evaluator.ts
@@ -7,12 +7,11 @@ export default class ManualEvaluator implements RuleEvaluator<JsonRule> {
   }
 
   async evaluateSingleResource(rule: Rule): Promise<RuleFinding> {
-    const finding = {
+    return {
       id: `${rule.id}/manual`,
       result: Result.SKIPPED,
       typename: 'manual',
       rule,
     } as RuleFinding
-    return finding
   }
 }
diff --git a/src/rules-engine/index.ts b/src/rules-engine/index.ts
@@ -6,7 +6,6 @@ import ManualEvaluator from './evaluators/manual-evaluator'
 import JsEvaluator from './evaluators/js-evaluator'
 import { RuleEvaluator } from './evaluators/rule-evaluator'
 import { Engine, ResourceData, Rule, RuleFinding } from './types'
-import { Entity } from '../types'
 
 export default class RulesProvider implements Engine {
   evaluators: RuleEvaluator<any>[] = []
@@ -167,8 +166,6 @@ export default class RulesProvider implements Engine {
     return [mainType, extensions]
   }
 
-  prepareMutations: (findings: RuleFinding[]) => Entity[]
-
   processRule = async (rule: Rule, data: unknown): Promise<RuleFinding[]> => {
     const res: any[] = []
     const dedupeIds = {}
diff --git a/src/rules-engine/types.ts b/src/rules-engine/types.ts
@@ -1,5 +1,3 @@
-import { Entity } from '..'
-
 export type ResourceData = {
   data: { [k: string]: any }
   resource: { id: string; [k: string]: any }
@@ -85,11 +83,4 @@ export interface Engine {
    * @returns An array of RuleFinding
    */
   processRule: (rule: Rule, data: any) => Promise<RuleFinding[]>
-
-  /**
-   * Transforms RuleFinding array into a mutation array for GraphQL
-   * @param findings resulted findings during rules execution
-   * @returns {Entity[]} Array of generated mutations
-   */
-  prepareMutations: (findings: RuleFinding[]) => Entity[]
 }
diff --git a/tests/rules-engine.test.ts b/tests/rules-engine.test.ts
@@ -4,6 +4,8 @@ import RulesProvider from '../src/rules-engine'
 import { Engine, Rule } from '../src/rules-engine/types'
 import ManualEvaluatorMock from './evaluators/manual-evaluator.test'
 import JSONEvaluatordMock from './evaluators/json-evaluator.test'
+import DgraphDataProcessor from '../src/rules-engine/data-processors/dgraph-data-processor'
+import DataProcessor from '../src/rules-engine/data-processors/data-processor'
 
 const typenameToFieldMap = {
   resourceA: 'querySchemaA',
@@ -14,12 +16,14 @@ const entityName = 'CIS'
 
 describe('RulesEngine', () => {
   let rulesEngine: Engine
+  let dataProcessor: DataProcessor
   beforeEach(() => {
     rulesEngine = new RulesProvider({
       providerName,
       entityName,
       typenameToFieldMap,
     })
+    dataProcessor = new DgraphDataProcessor(providerName, entityName)
   })
   it('Should pass getting the updated schema created dynamically using schemaTypeName and typenameToFieldMap fields', () => {
     const schema = rulesEngine.getSchema()
@@ -34,14 +38,13 @@ describe('RulesEngine', () => {
   })
 
   it('Should pass preparing the mutations to insert findings data given a RuleFindings array with Manual Rules', async () => {
-    await rulesEngine.processRule(
+    const findings = await rulesEngine.processRule(
       ManualEvaluatorMock.manualRule as Rule,
       undefined
     )
-    const findings = rulesEngine.prepareMutations([])
-    const [mutations] = findings
+    const [mutations] = dataProcessor.prepareMutations(findings)
 
-    expect(findings.length).toBe(3)
+    expect(findings.length).toBe(1)
     expect(mutations).toBeDefined()
     expect(mutations.data instanceof Array).toBeTruthy()
     expect(mutations.data.length).toBe(1)
@@ -54,19 +57,21 @@ describe('RulesEngine', () => {
   it('Should pass preparing the mutations to insert findings data given a RuleFindings array with Automated Rules', async () => {
     const resourceId = cuid()
     const resourceType = 'schemaA'
-    await rulesEngine.processRule(JSONEvaluatordMock.jsonRule as Rule, {
-      querySchemaA: [
-        {
-          id: resourceId,
-          __typename: resourceType,
-          value: 'automated',
-        },
-      ],
-    })
-    const findings = rulesEngine.prepareMutations([])
-    const [mutations] = findings
+    const findings = await rulesEngine.processRule(
+      JSONEvaluatordMock.jsonRule as Rule,
+      {
+        querySchemaA: [
+          {
+            id: resourceId,
+            __typename: resourceType,
+            value: 'automated',
+          },
+        ],
+      }
+    )
+    const [mutations] = dataProcessor.prepareMutations(findings)
 
-    expect(findings.length).toBe(4)
+    expect(findings.length).toBe(1)
     expect(mutations).toBeDefined()
     expect(mutations.data instanceof Object).toBeTruthy()
     expect(mutations.data.filter.id.eq).toBe(resourceId)
@@ -77,10 +82,10 @@ describe('RulesEngine', () => {
   it('Should pass preparing the mutations to insert with an empty findings array', () => {
     const data = []
 
-    const entities = rulesEngine.prepareMutations(data)
+    const entities = dataProcessor.prepareMutations(data)
 
     expect(entities).toBeDefined()
-    expect(entities.length).toBe(3)
+    expect(entities.length).toBe(2)
   })
 
   it('Should return an empty array processing a rule with no data', async () => {

Original file line number	Diff line number	Diff line change
`@@ -7,12 +7,11 @@ export default class ManualEvaluator implements RuleEvaluator<JsonRule> {`
`7`	`7`	`}`
`8`	`8`
`9`	`9`	`async evaluateSingleResource(rule: Rule): Promise<RuleFinding> {`
`10`		`- const finding = {`
	`10`	`+ return {`
`11`	`11`	id: `${rule.id}/manual`,
`12`	`12`	`result: Result.SKIPPED,`
`13`	`13`	`typename: 'manual',`
`14`	`14`	`rule,`
`15`	`15`	`} as RuleFinding`
`16`		`- return finding`
`17`	`16`	`}`
`18`	`17`	`}`