Merge branch 'feature/CG-891' into 'master'

feat(rulesEngine): Composite Rules

Closes CG-891

See merge request auto-cloud/cloudgraph/sdk!55

Author: Tyson Kunovsky

src/plugins/policyPack/index.ts

@@ -13,6 +13,8 @@ import {
 import RulesEngine from '../../rules-engine'
 import { Result, Rule, Severity } from '../../rules-engine/types'
 import Plugin, { ConfiguredPlugin, PluginManager } from '../types'
+import DgraphDataProcessor from '../../rules-engine/data-processors/dgraph-data-processor'
+import DataProcessor from '../../rules-engine/data-processors/data-processor'
 
 export default class PolicyPackPlugin extends Plugin {
   constructor({
@@ -61,6 +63,10 @@ export default class PolicyPackPlugin extends Plugin {
     }
   } = {}
 
+  private dataProcessors: {
+    [name: string]: DataProcessor
+  } = {}
+
   private async getPolicyPackPackage({
     policyPack,
     pluginManager,
@@ -130,6 +136,73 @@ export default class PolicyPackPlugin extends Plugin {
     }
   }
 
+  private async executeRule({
+    rules,
+    policyPack,
+    storageEngine,
+  }: {
+    rules: Rule[]
+    policyPack: string
+    storageEngine: StorageEngine
+  }): Promise<RuleFinding[]> {
+    const findings: RuleFinding[] = []
+
+    // Run rules:
+    for (const rule of rules) {
+      try {
+        if (rule.queries?.length > 0) {
+          const { queries, ...ruleMetadata } = rule
+          const subRules = queries.map(q => ({
+            ...q,
+            ...ruleMetadata,
+          }))
+
+          findings.push(
+            ...(await this.executeRule({
+              rules: subRules,
+              policyPack,
+              storageEngine,
+            }))
+          )
+        } else {
+          const { data } = rule.gql
+            ? await storageEngine.query(rule.gql)
+            : { data: undefined }
+          const results = (await this.policyPacksPlugins[
+            policyPack
+          ]?.engine?.processRule(rule, data)) as RuleFinding[]
+
+          findings.push(...results)
+        }
+      } catch (error) {
+        this.logger.error(
+          `Error processing rule ${rule.id} for ${policyPack} policy pack`
+        )
+        this.logger.debug(error)
+      }
+    }
+
+    return findings
+  }
+
+  // TODO: Generalize data processor moving storage module to SDK with its interfaces
+  private getDataProcessor({
+    entity,
+    provider,
+  }: {
+    entity: string
+    provider: string
+  }): DataProcessor {
+    const dataProcessorKey = `${provider}${entity}`
+    if (this.dataProcessors[dataProcessorKey]) {
+      return this.dataProcessors[dataProcessorKey]
+    }
+
+    const dataProcessor = new DgraphDataProcessor(provider, entity)
+    this.dataProcessors[dataProcessorKey] = dataProcessor
+    return dataProcessor
+  }
+
   async configure(
     pluginManager: PluginManager,
     plugins: ConfiguredPlugin[]
@@ -239,29 +312,20 @@ export default class PolicyPackPlugin extends Plugin {
           mergeSchemas(currentSchema, findingsSchema),
         ])
 
-        const findings: RuleFinding[] = []
-
-        // Run rules:
-        for (const rule of rules) {
-          try {
-            const { data } = rule.gql
-              ? await storageEngine.query(rule.gql)
-              : { data: undefined }
-            const results = (await this.policyPacksPlugins[
-              policyPack
-            ]?.engine?.processRule(rule, data)) as RuleFinding[]
-
-            findings.push(...results)
-          } catch (error) {
-            this.logger.error(
-              `Error processing rule ${rule.id} for ${policyPack} policy pack`
-            )
-            this.logger.debug(error)
-          }
-        }
+        const findings = await this.executeRule({
+          rules,
+          policyPack,
+          storageEngine,
+        })
+
+        // Data Processor
+        const dataProcessor = this.getDataProcessor({
+          entity,
+          provider: this.provider.name,
+        })
 
         // Prepare mutations
-        const mutations = engine?.prepareMutations(findings)
+        const mutations = dataProcessor.prepareMutations(findings)
 
         // Save connections
         processConnectionsBetweenEntities({
@@ -279,7 +343,6 @@ export default class PolicyPackPlugin extends Plugin {
         )
         this.logger.successSpinner('success')
 
-        // TODO: Use table to display results
         const results = findings.filter(
           finding => finding.result === Result.FAIL
         )

src/rules-engine/data-processors/data-processor.ts

@@ -0,0 +1,11 @@
+import { Entity } from '../../types'
+import { RuleFinding } from '../types'
+
+export default interface DataProcessor {
+  /**
+   * Transforms RuleFinding array into a mutation array for GraphQL
+   * @param findings resulted findings during rules execution
+   * @returns {Entity[]} Array of generated mutations
+   */
+  prepareMutations: (findings: RuleFinding[]) => Entity[]
+}

src/rules-engine/data-processors/dgraph-data-processor.ts

@@ -0,0 +1,146 @@
+import groupBy from 'lodash/groupBy'
+import isEmpty from 'lodash/isEmpty'
+import { Entity } from '../../types'
+import { RuleFinding } from '../types'
+import DataProcessor from './data-processor'
+
+export default class DgraphDataProcessor implements DataProcessor {
+  private readonly providerName
+
+  private readonly entityName
+
+  constructor(providerName: string, entityName: string) {
+    this.providerName = providerName
+    this.entityName = entityName
+  }
+
+  /**
+   * Prepare the mutations for overall provider findings
+   * @param findings RuleFinding array
+   * @returns A formatted Entity array
+   */
+  private prepareProviderMutations = (
+    findings: RuleFinding[] = []
+  ): Entity[] => {
+    // Prepare provider schema connections
+    return [
+      {
+        name: `${this.providerName}Findings`,
+        mutation: `
+        mutation($input: [Add${this.providerName}FindingsInput!]!) {
+          add${this.providerName}Findings(input: $input, upsert: true) {
+            numUids
+          }
+        }
+        `,
+        data: {
+          id: `${this.providerName}-provider`,
+        },
+      },
+      {
+        name: `${this.providerName}Findings`,
+        mutation: `mutation update${this.providerName}Findings($input: Update${this.providerName}FindingsInput!) {
+            update${this.providerName}Findings(input: $input) {
+              numUids
+            }
+          }
+          `,
+        data: {
+          filter: {
+            id: { eq: `${this.providerName}-provider` },
+          },
+          set: {
+            [`${this.entityName}Findings`]: findings.map(
+              ({ typename, ...rest }) => ({ ...rest })
+            ),
+          },
+        },
+      },
+    ]
+  }
+
+  private prepareProcessedMutations(findingsByType: {
+    [resource: string]: RuleFinding[]
+  }): Entity[] {
+    const mutations = []
+
+    for (const findingType in findingsByType) {
+      if (!isEmpty(findingType)) {
+        // Group Findings by resource
+        const findingsByResource = groupBy(
+          findingsByType[findingType],
+          'resourceId'
+        )
+
+        for (const resource in findingsByResource) {
+          if (resource) {
+            const data = (
+              (findingsByResource[resource] as RuleFinding[]) || []
+            ).map(({ typename, ...properties }) => properties)
+
+            // Create dynamically update mutations by resource
+            const updateMutation = {
+              name: `${this.providerName}${this.entityName}Findings`,
+              mutation: `mutation update${findingType}($input: Update${findingType}Input!) {
+                update${findingType}(input: $input) {
+                  numUids
+                }
+              }
+              `,
+              data: {
+                filter: {
+                  id: { eq: resource },
+                },
+                set: {
+                  [`${this.entityName}Findings`]: data,
+                },
+              },
+            }
+
+            mutations.push(updateMutation)
+          }
+        }
+      }
+    }
+
+    return mutations
+  }
+
+  private prepareManualMutations(findings: RuleFinding[] = []): Entity[] {
+    const manualFindings = findings.map(({ typename, ...finding }) => ({
+      ...finding,
+    }))
+    return manualFindings.length > 0
+      ? [
+          {
+            name: `${this.providerName}${this.entityName}Findings`,
+            mutation: `
+      mutation($input: [Add${this.providerName}${this.entityName}FindingsInput!]!) {
+        add${this.providerName}${this.entityName}Findings(input: $input, upsert: true) {
+          numUids
+        }
+      }
+      `,
+            data: manualFindings,
+          },
+        ]
+      : []
+  }
+
+  // TODO: Optimize generated mutations number
+  prepareMutations = (findings: RuleFinding[] = []): Entity[] => {
+    // Group Findings by schema type
+    const { manual, ...processedRules } = groupBy(findings, 'typename')
+
+    // Prepare processed rules mutations
+    const processedRulesData = this.prepareProcessedMutations(processedRules)
+
+    // Prepare manual mutations
+    const manualRulesData = this.prepareManualMutations(manual)
+
+    // Prepare provider mutations
+    const providerData = this.prepareProviderMutations(findings)
+
+    return [...manualRulesData, ...processedRulesData, ...providerData]
+  }
+}

src/rules-engine/evaluators/js-evaluator.ts

@@ -29,7 +29,6 @@ export default class JsEvaluator implements RuleEvaluator<JsRule> {
       typename: data.resource?.__typename, // eslint-disable-line no-underscore-dangle
       rule: ruleMetadata,
     } as RuleFinding
-
     return finding
   }
 }

src/rules-engine/evaluators/json-evaluator.ts

@@ -10,9 +10,9 @@ import {
   RuleResult,
   _ResourceData,
 } from '../types'
-import { RuleEvaluator } from './rule-evaluator'
 import AdditionalOperators from '../operators'
 import ComparisonOperators from '../operators/comparison'
+import { RuleEvaluator } from './rule-evaluator'
 
 export default class JsonEvaluator implements RuleEvaluator<JsonRule> {
   canEvaluate(rule: JsonRule): boolean {
@@ -35,7 +35,6 @@ export default class JsonEvaluator implements RuleEvaluator<JsonRule> {
       typename: data.resource?.__typename, // eslint-disable-line no-underscore-dangle
       rule: ruleMetadata,
     } as RuleFinding
-
     return finding
   }
 

src/rules-engine/evaluators/rule-evaluator.ts

@@ -3,5 +3,4 @@ import { ResourceData, Rule, RuleFinding } from '../types'
 export interface RuleEvaluator<K extends Rule> {
   canEvaluate: (rule: K) => boolean
   evaluateSingleResource: (rule: K, data?: ResourceData) => Promise<RuleFinding>
-  // @TODO complex rules can take a query and return an array of resourceId + Result
 }