Upgrade libddwaf native bindings (#5792)

Implement new major version for native-appsec native module, which provides an improved management system for WAF configuration updates and diagnostic result, used in telemetry logs and RC error reporting.

Author: CarlesDD

integration-tests/appsec/graphql.spec.js

@@ -109,7 +109,7 @@ describe('graphql', () => {
       assert.property(payload[1][0].metrics, '_dd.appsec.waf.duration')
       assert.propertyVal(payload[1][0].meta, 'appsec.event', 'true')
       assert.property(payload[1][0].meta, '_dd.appsec.json')
-      assert.propertyVal(payload[1][0].meta, '_dd.appsec.json', JSON.stringify(result))
+      assert.deepStrictEqual(JSON.parse(payload[1][0].meta['_dd.appsec.json']), result)
     })
 
     await axios({

package.json

@@ -86,7 +86,7 @@
   },
   "dependencies": {
     "@datadog/libdatadog": "0.7.0",
-    "@datadog/native-appsec": "8.5.2",
+    "@datadog/native-appsec": "9.0.0",
     "@datadog/native-iast-taint-tracking": "4.0.0",
     "@datadog/native-metrics": "3.1.1",
     "@datadog/pprof": "5.9.0",

packages/dd-trace/src/appsec/reporter.js

@@ -1,22 +1,26 @@
 'use strict'
 
+const dc = require('dc-polyfill')
+const zlib = require('zlib')
+
 const Limiter = require('../rate_limiter')
 const { storage } = require('../../../datadog-core')
 const web = require('../plugins/util/web')
 const { ipHeaderList } = require('../plugins/util/ip_extractor')
 const {
   incrementWafInitMetric,
   incrementWafUpdatesMetric,
+  incrementWafConfigErrorsMetric,
   incrementWafRequestsMetric,
   updateWafRequestsMetricTags,
   updateRaspRequestsMetricTags,
   updateRaspRuleSkippedMetricTags,
   updateRateLimitedMetric,
   getRequestMetrics
 } = require('./telemetry')
-const zlib = require('zlib')
 const { keepTrace } = require('../priority_sampler')
 const { ASM } = require('../standalone/product')
+const { DIAGNOSTIC_KEYS } = require('./waf/diagnostics')
 
 const REQUEST_HEADER_TAG_PREFIX = 'http.request.headers.'
 const RESPONSE_HEADER_TAG_PREFIX = 'http.response.headers.'
@@ -25,6 +29,8 @@ const COLLECTED_REQUEST_BODY_MAX_STRING_LENGTH = 4096
 const COLLECTED_REQUEST_BODY_MAX_DEPTH = 20
 const COLLECTED_REQUEST_BODY_MAX_ELEMENTS_PER_NODE = 256
 
+const telemetryLogCh = dc.channel('datadog:telemetry:log')
+
 // default limiter, configurable with setRateLimit()
 let limiter = new Limiter(100)
 
@@ -216,17 +222,64 @@ function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedRespons
 function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}, success = false) {
   if (success) {
     metricsQueue.set('_dd.appsec.waf.version', wafVersion)
-
-    metricsQueue.set('_dd.appsec.event_rules.loaded', diagnosticsRules.loaded?.length || 0)
-    metricsQueue.set('_dd.appsec.event_rules.error_count', diagnosticsRules.failed?.length || 0)
-    if (diagnosticsRules.failed?.length) {
-      metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors))
-    }
   }
 
   incrementWafInitMetric(wafVersion, rulesVersion, success)
 }
 
+function logWafDiagnosticMessage (product, rcConfigId, configKey, message, level) {
+  const tags =
+    `log_type:rc::${product.toLowerCase()}::diagnostic,appsec_config_key:${configKey},rc_config_id:${rcConfigId}`
+  telemetryLogCh.publish({
+    message,
+    level,
+    tags
+  })
+}
+
+function reportWafConfigUpdate (product, rcConfigId, diagnostics, wafVersion) {
+  if (diagnostics.error) {
+    logWafDiagnosticMessage(product, rcConfigId, '', diagnostics.error, 'ERROR')
+    incrementWafConfigErrorsMetric(wafVersion, diagnostics.ruleset_version)
+  }
+
+  for (const configKey of DIAGNOSTIC_KEYS) {
+    const configDiagnostics = diagnostics[configKey]
+    if (!configDiagnostics) continue
+
+    if (configDiagnostics.error) {
+      logWafDiagnosticMessage(product, rcConfigId, configKey, configDiagnostics.error, 'ERROR')
+      incrementWafConfigErrorsMetric(wafVersion, diagnostics.ruleset_version)
+      continue
+    }
+
+    if (configDiagnostics.errors) {
+      for (const [errorMessage, errorIds] of Object.entries(configDiagnostics.errors)) {
+        logWafDiagnosticMessage(
+          product,
+          rcConfigId,
+          configKey,
+          `"${errorMessage}": ${JSON.stringify(errorIds)}`,
+          'ERROR'
+        )
+        incrementWafConfigErrorsMetric(wafVersion, diagnostics.ruleset_version)
+      }
+    }
+
+    if (configDiagnostics.warnings) {
+      for (const [warningMessage, warningIds] of Object.entries(configDiagnostics.warnings)) {
+        logWafDiagnosticMessage(
+          product,
+          rcConfigId,
+          configKey,
+          `"${warningMessage}": ${JSON.stringify(warningIds)}`,
+          'WARN'
+        )
+      }
+    }
+  }
+}
+
 function reportMetrics (metrics, raspRule) {
   const store = storage('legacy').getStore()
   const rootSpan = store?.req && web.root(store.req)
@@ -485,6 +538,7 @@ module.exports = {
   filterExtendedHeaders,
   formatHeaderName,
   reportWafInit,
+  reportWafConfigUpdate,
   reportMetrics,
   reportAttack,
   reportWafUpdate: incrementWafUpdatesMetric,