Merge pull request #23 from cloudgraphdev/beta

RELEASE 0.21.0

Author: tyler-dunkel

.github/workflows/publish.yml

@@ -41,9 +41,6 @@ jobs:
         # See https://github.com/actions/setup-node/blob/main/docs/advanced-usage.md#use-private-packages
         run: yarn install --frozen-lockfile --prefer-offline --ignore-scripts
 
-      - name: Install jq for tests
-        run: node -r node-jq/scripts/install-binary.js
-
       - name: Build
         run: yarn prepack
 

CHANGELOG.md

@@ -1,3 +1,51 @@
+# [0.21.0-beta.1](https://github.com/cloudgraphdev/sdk/compare/0.20.0...0.21.0-beta.1) (2022-07-06)
+
+
+### Bug Fixes
+
+* adapt CLI to pass data instead of queries ([dbaa7ce](https://github.com/cloudgraphdev/sdk/commit/dbaa7ced67870863792cedaaf80b965c6d11cbf0))
+* remove node-jq implementation ([c614842](https://github.com/cloudgraphdev/sdk/commit/c6148429e056cd64cc3055f861b597e2dbcdbb09))
+* Rollback missing resources status feature ([5eb10e9](https://github.com/cloudgraphdev/sdk/commit/5eb10e9c610e92ec608acbac1fdefb4a84ea109c))
+
+
+### Features
+
+* adapt CLI to pass data instead of queries ([1d73587](https://github.com/cloudgraphdev/sdk/commit/1d735872a6703e3880b26df8ee8d33982d56c3f1))
+* Added unit tests ([0d14267](https://github.com/cloudgraphdev/sdk/commit/0d14267c357307b8812d4d35faa2a50bc73437a7))
+* Extend rule engine to include missing checks when the resource does not exist ([79f8dd3](https://github.com/cloudgraphdev/sdk/commit/79f8dd3c9ffa149c431feb09f9b087a8b92e5250))
+* Move getLinkedData method to utils folder ([171d937](https://github.com/cloudgraphdev/sdk/commit/171d937ba7d4f1b6b9c0f878bd58288724ba138e))
+* remove unused storageEngine param ([a9a4ccb](https://github.com/cloudgraphdev/sdk/commit/a9a4ccb3613e7cf3dbc08a1e74bddef9d082fbc4))
+
+# [0.21.0-alpha.3](https://github.com/cloudgraphdev/sdk/compare/0.21.0-alpha.2...0.21.0-alpha.3) (2022-07-06)
+
+
+### Bug Fixes
+
+* remove node-jq implementation ([c614842](https://github.com/cloudgraphdev/sdk/commit/c6148429e056cd64cc3055f861b597e2dbcdbb09))
+* Rollback missing resources status feature ([5eb10e9](https://github.com/cloudgraphdev/sdk/commit/5eb10e9c610e92ec608acbac1fdefb4a84ea109c))
+
+# [0.21.0-alpha.2](https://github.com/cloudgraphdev/sdk/compare/0.21.0-alpha.1...0.21.0-alpha.2) (2022-06-30)
+
+
+### Bug Fixes
+
+* adapt CLI to pass data instead of queries ([dbaa7ce](https://github.com/cloudgraphdev/sdk/commit/dbaa7ced67870863792cedaaf80b965c6d11cbf0))
+
+
+### Features
+
+* adapt CLI to pass data instead of queries ([1d73587](https://github.com/cloudgraphdev/sdk/commit/1d735872a6703e3880b26df8ee8d33982d56c3f1))
+* Move getLinkedData method to utils folder ([171d937](https://github.com/cloudgraphdev/sdk/commit/171d937ba7d4f1b6b9c0f878bd58288724ba138e))
+* remove unused storageEngine param ([a9a4ccb](https://github.com/cloudgraphdev/sdk/commit/a9a4ccb3613e7cf3dbc08a1e74bddef9d082fbc4))
+
+# [0.21.0-alpha.1](https://github.com/cloudgraphdev/sdk/compare/0.20.0...0.21.0-alpha.1) (2022-06-29)
+
+
+### Features
+
+* Added unit tests ([0d14267](https://github.com/cloudgraphdev/sdk/commit/0d14267c357307b8812d4d35faa2a50bc73437a7))
+* Extend rule engine to include missing checks when the resource does not exist ([79f8dd3](https://github.com/cloudgraphdev/sdk/commit/79f8dd3c9ffa149c431feb09f9b087a8b92e5250))
+
 # [0.20.0](https://github.com/cloudgraphdev/sdk/compare/0.19.0...0.20.0) (2022-06-09)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cloudgraph/sdk",
-  "version": "0.20.0",
+  "version": "0.21.0-beta.1",
   "description": "SDK for cloud graph providers and cli",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -29,7 +29,6 @@
     "inquirer": "^8.1.2",
     "jsonpath": "^1.1.1",
     "lodash": "^4.17.21",
-    "node-jq": "^2.3.0",
     "ora": "^5.4.1"
   },
   "devDependencies": {

src/plugins/policyPack/index.ts

@@ -15,6 +15,7 @@ import { Result, Rule, Severity } from '../../rules-engine/types'
 import Plugin, { ConfiguredPlugin, PluginManager } from '../types'
 import DgraphDataProcessor from '../../rules-engine/data-processors/dgraph-data-processor'
 import DataProcessor from '../../rules-engine/data-processors/data-processor'
+import getLinkedData from '../../utils/data'
 
 export default class PolicyPackPlugin extends Plugin {
   constructor({
@@ -137,13 +138,13 @@ export default class PolicyPackPlugin extends Plugin {
   }
 
   private async executeRule({
+    data,
     rules,
     policyPack,
-    storageEngine,
   }: {
+    data: any,
     rules: Rule[]
     policyPack: string
-    storageEngine: StorageEngine
   }): Promise<RuleFinding[]> {
     const findings: RuleFinding[] = []
 
@@ -159,18 +160,16 @@ export default class PolicyPackPlugin extends Plugin {
 
           findings.push(
             ...(await this.executeRule({
+              data,
               rules: subRules,
               policyPack,
-              storageEngine,
             }))
           )
         } else {
-          const { data } = rule.gql
-            ? await storageEngine.query(rule.gql)
-            : { data: undefined }
+          const ruleData = rule.gql ? data : undefined
           const results = (await this.policyPacksPlugins[
             policyPack
-          ]?.engine?.processRule(rule, data)) as RuleFinding[]
+          ]?.engine?.processRule(rule, ruleData)) as RuleFinding[]
 
           findings.push(...results)
         }
@@ -277,10 +276,12 @@ export default class PolicyPackPlugin extends Plugin {
   async execute({
     storageRunning,
     storageEngine,
+    providerData,
     processConnectionsBetweenEntities,
   }: {
     storageRunning: boolean
     storageEngine: StorageEngine
+    providerData: ProviderData
     processConnectionsBetweenEntities: (props: {
       provider?: string
       providerData: ProviderData
@@ -314,10 +315,13 @@ export default class PolicyPackPlugin extends Plugin {
           mergeSchemas(currentSchema, findingsSchema),
         ])
 
+        // Format metadata and link connections
+        const linkedData = getLinkedData(providerData, this.provider.schemasMap)
+
         const findings = await this.executeRule({
+          data: linkedData,
           rules,
           policyPack,
-          storageEngine,
         })
 
         // Prepare mutations

src/rules-engine/evaluators/json-evaluator.ts

@@ -1,5 +1,5 @@
 import lodash from 'lodash'
-import * as jqNode from 'node-jq'
+
 import {
   Condition,
   JsonRule,
@@ -20,9 +20,12 @@ export default class JsonEvaluator implements RuleEvaluator<JsonRule> {
   }
 
   async evaluateSingleResource(
-    { id, conditions, severity }: JsonRule,
+    { id, conditions, severity, exclude }: JsonRule,
     data: ResourceData
   ): Promise<RuleFinding> {
+    if (exclude && (await this.evaluateCondition(exclude, data))) {
+      return
+    }
     const result = (await this.evaluateCondition(conditions, data))
       ? RuleResult.MATCHES
       : RuleResult.DOESNT_MATCH
@@ -119,10 +122,9 @@ export default class JsonEvaluator implements RuleEvaluator<JsonRule> {
     _data: _ResourceData
   ): Promise<number | boolean> {
     const condition = { ..._condition }
-    const { path, value, jq: jqQuery } = condition
+    const { path, value } = condition
     delete condition.path
     delete condition.value
-    delete condition.jq
     // remaining field should be the op name
     const op = Object.keys(condition)[0] //
     const operator = this.operators[op]
@@ -155,24 +157,6 @@ export default class JsonEvaluator implements RuleEvaluator<JsonRule> {
       firstArg = value
     }
 
-    if (firstArg && jqQuery) {
-      firstArg = await this.runJq(firstArg, jqQuery)
-      lodash.set(data.data, data.elementPath, firstArg)
-    }
-
     return operator(firstArg, otherArgs, data)
   }
-
-  async runJq(data: unknown, jqQuery: string): Promise<unknown> {
-    try {
-      const json = (await jqNode.run(jqQuery, data, {
-        input: 'json',
-        output: 'json',
-      })) as unknown
-
-      return json || data
-    } catch (e) {
-      return data
-    }
-  }
 }

src/rules-engine/index.ts

@@ -51,6 +51,8 @@ export default class RulesProvider implements Engine {
   ): Promise<RuleFinding> => {
     const finding = await evaluator.evaluateSingleResource(rule, data)
 
+    if (!finding) return
+    
     // Inject extra fields
     for (const field of this.dataProcessor.extraFields) {
       finding[field] = data.resource[field]
@@ -91,16 +93,16 @@ export default class RulesProvider implements Engine {
     const resourcePaths = data ? jsonpath.nodes(data, rule.resource) : []
     const evaluator = this.getRuleEvaluator(rule)
 
+    if (!evaluator) {
+      return []
+    }
+
     if (isEmpty(data) && evaluator instanceof ManualEvaluator) {
       // Returned Manual Rule Response
       res.push(await evaluator.evaluateSingleResource(rule))
       return res
     }
 
-    if (!evaluator) {
-      return []
-    }
-
     // @NOTE: here we can evaluate things such as Data being empty (may imply rule to pass)
     // or if we have no resources, or none pass, we can decide on that rule (+ some rule field)
     for (let i = 0; i < resourcePaths.length; i++) {

src/rules-engine/operators/common.ts

@@ -13,6 +13,8 @@ export default {
       return shouldBeEmpty ? hasKeys === 0 : hasKeys > 0
     }
 
-    return false
+    return (
+      (data === null || data === undefined || data === '') === shouldBeEmpty
+    )
   },
 }

src/rules-engine/types.ts

@@ -9,7 +9,6 @@ export type ResourceData = {
 export type Condition = {
   path?: string
   value?: string | number | Condition | (string | number)[]
-  jq?: string
   [operationId: string]: any
 }
 
@@ -65,6 +64,7 @@ export interface RuleFinding extends Finding {
 
 export interface JsonRule extends Rule {
   conditions: Condition
+  exclude?: Condition
 }
 
 export interface JsRule extends Rule {

src/utils/data.ts

@@ -0,0 +1,58 @@
+import { ProviderData, SchemaMap } from '../types'
+
+const getLinkedData = (providerData: ProviderData, schemasMap?: SchemaMap): any => {
+  const linkedData = {}
+  const allEntities = providerData?.entities || []
+  const allConnections = providerData?.connections || {}
+  const entitiesById: { [key: string]: any } = {}
+
+  for (const entity of allEntities) {
+    // AddawsEc2Input! => queryawsEc2
+    const mutationName = /(?<=\[)(.*?)(?=\])/
+      .exec(entity.mutation as any)[0]
+      .replace('Add', 'query')
+      .replace('Input!', '')
+
+    linkedData[mutationName] = entity.data
+
+    for (const entityData of entity.data) {
+      entitiesById[entityData.id] = entityData
+      // eslint-disable-next-line no-underscore-dangle
+      entityData.__typename = mutationName.replace('query', '') // or entity.name?
+    }
+  }
+
+  // connect data on a second pass
+  for (const entityId of Object.keys(allConnections)) {
+    const entityConnections = allConnections[entityId]
+    const entity = entitiesById[entityId]
+    if (!entity) {
+      // eslint-disable-next-line no-continue
+      continue
+    }
+    for (const conn of entityConnections) {
+      const targetEntity = entitiesById[conn.id]
+      if (!targetEntity) {
+        // eslint-disable-next-line no-continue
+        continue
+      }
+      if (conn.relation === 'child') {
+        if (!entity[conn.field]) {
+          entity[conn.field] = []
+        }
+        entity[conn.field].push(targetEntity)
+        // inverse relation
+        // eslint-disable-next-line no-underscore-dangle
+        const inverseConnField = schemasMap && schemasMap[entity.__typename] || 'account' // @TODO: account doesn't have a name
+        if (!targetEntity[inverseConnField]) {
+          targetEntity[inverseConnField] = []
+        }
+        targetEntity[inverseConnField].push(entity)
+      } // else parent relation.. is not used atm
+    }
+  }
+
+  return linkedData
+}
+
+export default getLinkedData