feat: add automatic heap snapshots to help debug memory issues (#5920)

Author: rochdev

packages/dd-trace/src/config.js

@@ -529,6 +529,9 @@ class Config {
     defaults['grpc.client.error.statuses'] = GRPC_CLIENT_ERROR_STATUSES
     defaults['grpc.server.error.statuses'] = GRPC_SERVER_ERROR_STATUSES
     defaults.headerTags = []
+    defaults['heapSnapshot.count'] = 0
+    defaults['heapSnapshot.destination'] = ''
+    defaults['heapSnapshot.interval'] = 3600
     defaults.hostname = '127.0.0.1'
     defaults['iast.dbRowsToTaint'] = 1
     defaults['iast.deduplicationEnabled'] = true
@@ -713,6 +716,9 @@ class Config {
       DD_GRPC_CLIENT_ERROR_STATUSES,
       DD_GRPC_SERVER_ERROR_STATUSES,
       JEST_WORKER_ID,
+      DD_HEAP_SNAPSHOT_COUNT,
+      DD_HEAP_SNAPSHOT_DESTINATION,
+      DD_HEAP_SNAPSHOT_INTERVAL,
       DD_IAST_DB_ROWS_TO_TAINT,
       DD_IAST_DEDUPLICATION_ENABLED,
       DD_IAST_ENABLED,
@@ -896,6 +902,9 @@ class Config {
     this._setIntegerRangeSet(env, 'grpc.client.error.statuses', DD_GRPC_CLIENT_ERROR_STATUSES)
     this._setIntegerRangeSet(env, 'grpc.server.error.statuses', DD_GRPC_SERVER_ERROR_STATUSES)
     this._setArray(env, 'headerTags', DD_TRACE_HEADER_TAGS)
+    env['heapSnapshot.count'] = maybeInt(DD_HEAP_SNAPSHOT_COUNT)
+    this._setString(env, 'heapSnapshot.destination', DD_HEAP_SNAPSHOT_DESTINATION)
+    env['heapSnapshot.interval'] = maybeInt(DD_HEAP_SNAPSHOT_INTERVAL)
     this._setString(env, 'hostname', DD_AGENT_HOST)
     env['iast.dbRowsToTaint'] = maybeInt(DD_IAST_DB_ROWS_TO_TAINT)
     this._setBoolean(env, 'iast.deduplicationEnabled', DD_IAST_DEDUPLICATION_ENABLED)

packages/dd-trace/src/heap_snapshots.js

@@ -0,0 +1,58 @@
+'use strict'
+
+const { join } = require('path')
+const { setImmediate, setTimeout } = require('timers/promises')
+const { format } = require('util')
+const { writeHeapSnapshot } = require('v8')
+const { threadId } = require('worker_threads')
+const log = require('./log')
+
+async function scheduleSnapshot (config, total) {
+  if (total > config.heapSnapshot.count) return
+
+  await setTimeout(config.heapSnapshot.interval * 1000, null, { ref: false })
+  await clearMemory()
+  writeHeapSnapshot(getName(config.heapSnapshot.destination))
+  await scheduleSnapshot(config, total + 1)
+}
+
+async function clearMemory () {
+  if (!globalThis.gc) return
+  globalThis.gc()
+  await setImmediate()
+  globalThis.gc() // Run full GC a second time for anything missed in first GC.
+}
+
+function pad (value) {
+  return String(value).padStart(2, 0)
+}
+
+function getName (destination) {
+  const date = new Date()
+  const filename = format(
+    'Heap-%s%s%s-%s%s%s-%s-%s.heapsnapshot',
+    date.getFullYear(),
+    pad(date.getMonth()),
+    pad(date.getDate()),
+    pad(date.getHours()),
+    pad(date.getMinutes()),
+    pad(date.getSeconds()),
+    process.pid,
+    threadId
+  )
+
+  return join(destination, filename)
+}
+
+module.exports = {
+  async start (config) {
+    const destination = config.heapSnapshot.destination
+
+    try {
+      await scheduleSnapshot(config, 1)
+      log.debug('Wrote heap snapshots to %s.', destination)
+    } catch (e) {
+      log.error('Failed to write heap snapshots to %s.', destination, e)
+    }
+  }
+}

packages/dd-trace/src/proxy.js

@@ -100,6 +100,10 @@ class Tracer extends NoopProxy {
         require('./crashtracking').start(config)
       }
 
+      if (config.heapSnapshot.count > 0) {
+        require('./heap_snapshots').start(config)
+      }
+
       telemetry.start(config, this._pluginManager)
 
       if (config.dogstatsd) {

packages/dd-trace/src/supported-configurations.json

@@ -78,6 +78,9 @@
     "DD_GIT_PULL_REQUEST_BASE_BRANCH_SHA": ["A"],
     "DD_GRPC_CLIENT_ERROR_STATUSES": ["A"],
     "DD_GRPC_SERVER_ERROR_STATUSES": ["A"],
+    "DD_HEAP_SNAPSHOT_COUNT": ["A"],
+    "DD_HEAP_SNAPSHOT_INTERVAL": ["A"],
+    "DD_HEAP_SNAPSHOT_DESTINATION": ["A"],
     "DD_IAST_DB_ROWS_TO_TAINT": ["A"],
     "DD_IAST_DEDUPLICATION_ENABLED": ["A"],
     "DD_IAST_ENABLED": ["A"],

packages/dd-trace/test/config.spec.js

@@ -303,6 +303,9 @@ describe('Config', () => {
     expect(config).to.have.property('flushMinSpans', 1000)
     expect(config.grpc.client.error.statuses).to.deep.equal(GRPC_CLIENT_ERROR_STATUSES)
     expect(config.grpc.server.error.statuses).to.deep.equal(GRPC_SERVER_ERROR_STATUSES)
+    expect(config).to.have.nested.property('heapSnapshot.count', 0)
+    expect(config).to.have.nested.property('heapSnapshot.destination', '')
+    expect(config).to.have.nested.property('heapSnapshot.interval', 3600)
     expect(config).to.have.nested.property('iast.enabled', false)
     expect(config).to.have.nested.property('iast.redactionEnabled', true)
     expect(config).to.have.nested.property('iast.redactionNamePattern', null)
@@ -557,6 +560,9 @@ describe('Config', () => {
     process.env.DD_ENV = 'test'
     process.env.DD_GRPC_CLIENT_ERROR_STATUSES = '3,13,400-403'
     process.env.DD_GRPC_SERVER_ERROR_STATUSES = '3,13,400-403'
+    process.env.DD_HEAP_SNAPSHOT_COUNT = '1'
+    process.env.DD_HEAP_SNAPSHOT_DESTINATION = '/tmp'
+    process.env.DD_HEAP_SNAPSHOT_INTERVAL = '1800'
     process.env.DD_IAST_DB_ROWS_TO_TAINT = 2
     process.env.DD_IAST_DEDUPLICATION_ENABLED = false
     process.env.DD_IAST_ENABLED = 'true'
@@ -675,6 +681,9 @@ describe('Config', () => {
     expect(config.grpc.client.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403])
     expect(config.grpc.server.error.statuses).to.deep.equal([3, 13, 400, 401, 402, 403])
     expect(config).to.have.property('hostname', 'agent')
+    expect(config).to.have.nested.property('heapSnapshot.count', 1)
+    expect(config).to.have.nested.property('heapSnapshot.destination', '/tmp')
+    expect(config).to.have.nested.property('heapSnapshot.interval', 1800)
     expect(config).to.have.nested.property('iast.dbRowsToTaint', 2)
     expect(config).to.have.nested.property('iast.deduplicationEnabled', false)
     expect(config).to.have.nested.property('iast.enabled', true)

packages/dd-trace/test/heap_snapshots.spec.js

@@ -0,0 +1,35 @@
+'use strict'
+
+require('./setup/tap')
+
+const { mkdtempSync, readdirSync } = require('fs')
+const { tmpdir } = require('os')
+const { join } = require('path')
+const { threadId } = require('worker_threads')
+const { start } = require('../src/heap_snapshots')
+
+const destination = mkdtempSync(join(tmpdir(), 'dd-trace-heap-snapshot-'))
+
+describe('Heap Snapshots', () => {
+  it('should take heap snapshots over time', async () => {
+    // Keep process alive since `start` uses an unref timer.
+    const interval = setInterval(() => {}, 1000)
+
+    await start({
+      heapSnapshot: {
+        count: 3,
+        destination,
+        interval: 1
+      }
+    })
+
+    clearInterval(interval)
+
+    const pattern = new RegExp(`^Heap-\\d{8}-\\d{6}-${process.pid}-${threadId}\\.heapsnapshot$`)
+    const files = readdirSync(destination)
+
+    expect(files).to.have.length(3)
+    expect(files[0]).to.match(pattern)
+    expect(files[1]).to.match(pattern)
+  })
+})

packages/dd-trace/test/proxy.spec.js

@@ -139,7 +139,8 @@ describe('TracerProxy', () => {
         enabled: false
       },
       configure: sinon.spy(),
-      llmobs: {}
+      llmobs: {},
+      heapSnapshot: {}
     }
     Config = sinon.stub().returns(config)