Add telemetry for number of live CPED native sample context objects

Author: szegedi

integration-tests/profiler/profiler.spec.js

@@ -650,7 +650,7 @@ describe('profiler', () => {
     })
   })
 
-  context('Profiler API telemetry', () => {
+  context('Profiler telemetry', () => {
     beforeEach(async () => {
       agent = await new FakeAgent().start()
     })
@@ -678,21 +678,20 @@ describe('profiler', () => {
         const pp = payload.payload
         assert.equal(pp.namespace, 'profilers')
         const series = pp.series
-        assert.lengthOf(series, 2)
-        assert.equal(series[0].metric, 'profile_api.requests')
-        assert.equal(series[0].type, 'count')
+        const requests = series.find(s => s.metric === 'profile_api.requests')
+        assert.equal(requests.type, 'count')
         // There's a race between metrics and on-shutdown profile, so metric
         // value will be between 1 and 3
-        requestCount = series[0].points[0][1]
+        requestCount = requests.points[0][1]
         assert.isAtLeast(requestCount, 1)
         assert.isAtMost(requestCount, 3)
 
-        assert.equal(series[1].metric, 'profile_api.responses')
-        assert.equal(series[1].type, 'count')
-        assert.include(series[1].tags, 'status_code:200')
+        const responses = series.find(s => s.metric === 'profile_api.responses')
+        assert.equal(responses.type, 'count')
+        assert.include(responses.tags, 'status_code:200')
 
         // Same number of requests and responses
-        assert.equal(series[1].points[0][1], requestCount)
+        assert.equal(responses.points[0][1], requestCount)
       }, 'generate-metrics', timeout)
 
       const checkDistributions = agent.assertTelemetryReceived(({ _, payload }) => {
@@ -713,6 +712,34 @@ describe('profiler', () => {
       // Same number of requests and points
       assert.equal(requestCount, pointsCount)
     })
+
+    it('sends wall profiler sample context telemetry', async function () {
+      if (satisfies(process.versions.node, '<24.0.0')) {
+        this.skip() // Wall profiler context count telemetry is not supported in Node < 24
+      }
+      proc = fork(profilerTestFile, {
+        cwd,
+        env: {
+          DD_TRACE_AGENT_PORT: agent.port,
+          DD_PROFILING_ENABLED: 1,
+          DD_PROFILING_UPLOAD_PERIOD: 1,
+          DD_PROFILING_USE_ASYNC_CONTEXT_FRAME: 1,
+          DD_TELEMETRY_HEARTBEAT_INTERVAL: 1, // every second
+          TEST_DURATION_MS: 1500
+        }
+      })
+
+      const checkMetrics = agent.assertTelemetryReceived(({ _, payload }) => {
+        const pp = payload.payload
+        assert.equal(pp.namespace, 'profilers')
+        const sampleContexts = pp.series.find(s => s.metric === 'wall.sample_contexts')
+        assert.isDefined(sampleContexts)
+        assert.equal(sampleContexts.type, 'gauge')
+        assert.isAtLeast(sampleContexts.points[0][1], 1)
+      }, 'generate-metrics', timeout)
+
+      await Promise.all([checkProfiles(agent, proc, timeout), checkMetrics])
+    })
   })
 
   function forkSsi (args) {

packages/dd-trace/src/profiler.js

@@ -10,6 +10,8 @@ module.exports = {
   start: config => {
     const { service, version, env, url, hostname, port, tags, repositoryUrl, commitSHA, injectionEnabled } = config
     const { enabled, sourceMap, exporters } = config.profiling
+    const { heartbeatInterval } = config.telemetry
+
     const logger = {
       debug: (message) => log.debug(message),
       info: (message) => log.info(message),
@@ -39,7 +41,8 @@ module.exports = {
       repositoryUrl,
       commitSHA,
       libraryInjected,
-      activation
+      activation,
+      heartbeatInterval
     })
   },
 

packages/dd-trace/src/profiling/config.js

@@ -232,6 +232,8 @@ class Config {
       }
     }
 
+    this.heartbeatInterval = options.heartbeatInterval || 60 * 1000 // 1 minute
+
     this.profilers = ensureProfilers(profilers, this)
   }
 }

packages/dd-trace/src/profiling/profilers/wall.js

@@ -108,6 +108,7 @@ class NativeWallProfiler {
     this._timelineEnabled = !!options.timelineEnabled
     this._cpuProfilingEnabled = !!options.cpuProfilingEnabled
     this._useAsyncContextFrame = !!options.useAsyncContextFrame
+    this._telemetryHeartbeatIntervalMillis = options.heartbeatInterval || 60 * 1e3 // 60 seconds
 
     // We need to capture span data into the sample context for either code hotspots
     // or endpoint collection.
@@ -177,7 +178,9 @@ class NativeWallProfiler {
 
         ensureChannelsActivated(this._useAsyncContextFrame)
 
-        if (!this._useAsyncContextFrame) {
+        if (this._useAsyncContextFrame) {
+          this._setupTelemetryMetrics()
+        } else {
           beforeCh.subscribe(this._enter)
         }
         enterCh.subscribe(this._enter)
@@ -188,6 +191,16 @@ class NativeWallProfiler {
     this._started = true
   }
 
+  _setupTelemetryMetrics () {
+    const contextCountGauge = profilerTelemetryMetrics.gauge('wall.sample_contexts')
+
+    const that = this
+    this._contextCountGaugeUpdater = setInterval(() => {
+      contextCountGauge.mark(that._profilerState[kCPEDContextCount])
+    }, that._telemetryHeartbeatIntervalMillis)
+    this._contextCountGaugeUpdater.unref()
+  }
+
   _enter () {
     if (!this._started) return
 
@@ -315,6 +328,7 @@ class NativeWallProfiler {
         this._reportV8bug(v8BugDetected === 1)
       }
     } else {
+      clearInterval(this._contextCountGaugeUpdater)
       if (this._captureSpanData) {
         if (!this._useAsyncContextFrame) {
           beforeCh.unsubscribe(this._enter)