From e975778d146489c836d6f53635a5fac3a8ddd5e6 Mon Sep 17 00:00:00 2001 From: rochdev Date: Thu, 15 Dec 2022 20:53:23 -0500 Subject: [PATCH 01/19] add benchmark and incomplete implementation for event based tracing --- benchmark/sirun/plugin-koa/README.md | 3 + benchmark/sirun/plugin-koa/agent.js | 22 ++ .../plugin-koa/internal-tracer/client.js | 92 +++++ .../plugin-koa/internal-tracer/encoder.js | 362 ++++++++++++++++++ .../sirun/plugin-koa/internal-tracer/id.js | 126 ++++++ .../sirun/plugin-koa/internal-tracer/index.js | 5 + .../sirun/plugin-koa/internal-tracer/patch.js | 32 ++ .../plugin-koa/internal-tracer/storage.js | 20 + .../sirun/plugin-koa/internal-tracer/trace.js | 45 +++ benchmark/sirun/plugin-koa/meta.json | 26 ++ benchmark/sirun/plugin-koa/server.js | 44 +++ benchmark/sirun/plugin-koa/setup.js | 13 + 12 files changed, 790 insertions(+) create mode 100644 benchmark/sirun/plugin-koa/README.md create mode 100644 benchmark/sirun/plugin-koa/agent.js create mode 100644 benchmark/sirun/plugin-koa/internal-tracer/client.js create mode 100644 benchmark/sirun/plugin-koa/internal-tracer/encoder.js create mode 100644 benchmark/sirun/plugin-koa/internal-tracer/id.js create mode 100644 benchmark/sirun/plugin-koa/internal-tracer/index.js create mode 100644 benchmark/sirun/plugin-koa/internal-tracer/patch.js create mode 100644 benchmark/sirun/plugin-koa/internal-tracer/storage.js create mode 100644 benchmark/sirun/plugin-koa/internal-tracer/trace.js create mode 100644 benchmark/sirun/plugin-koa/meta.json create mode 100644 benchmark/sirun/plugin-koa/server.js create mode 100644 benchmark/sirun/plugin-koa/setup.js diff --git a/benchmark/sirun/plugin-koa/README.md b/benchmark/sirun/plugin-koa/README.md new file mode 100644 index 00000000000..9bba28b0d38 --- /dev/null +++ b/benchmark/sirun/plugin-koa/README.md @@ -0,0 +1,3 @@ +This creates 100,000 HTTP requests to a Koa server. + +The variants are with the tracer, without it, and with a new internal tracer. diff --git a/benchmark/sirun/plugin-koa/agent.js b/benchmark/sirun/plugin-koa/agent.js new file mode 100644 index 00000000000..da42f07a823 --- /dev/null +++ b/benchmark/sirun/plugin-koa/agent.js @@ -0,0 +1,22 @@ +'use strict' + +const express = require('express') +const bodyParser = require('body-parser') + +const app = express() + +let requests = 0 +let bytes = 0 + +app.use(bodyParser.raw({ limit: '50mb', type: () => true })) +app.use('*', (req, res) => { + requests++ + bytes += req.body.length + + console.log(`Requests: ${requests}`) // eslint-disable-line no-console + console.log(`Bytes: ${bytes}`) // eslint-disable-line no-console + + res.status(200).send() +}) + +app.listen(8126) diff --git a/benchmark/sirun/plugin-koa/internal-tracer/client.js b/benchmark/sirun/plugin-koa/internal-tracer/client.js new file mode 100644 index 00000000000..3b172a015ae --- /dev/null +++ b/benchmark/sirun/plugin-koa/internal-tracer/client.js @@ -0,0 +1,92 @@ +'use strict' + +const http = require('http') +const https = require('https') +const { dockerId, storage } = require('../../../../packages/datadog-core') +const tracerVersion = require('../../../../package.json').version + +const httpAgent = new http.Agent({ keepAlive: true, maxSockets: 1 }) +const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 1 }) + +const DD_TRACE_AGENT_URL = process.env.DD_TRACE_AGENT_URL || process.env.DD_TRACE_URL + +class Client { + request (options, done) { + if (options.count === 0) return + + const url = new URL(DD_TRACE_AGENT_URL || 'http://127.0.0.1:8126') + const isSecure = url.protocol === 'https:' + const isUnix = url.protocol === 'unix:' + const client = isSecure ? https : http + const agent = isSecure ? httpsAgent : httpAgent + const data = options.data + const timeout = 2000 + const httpOptions = { + agent, + protocol: url.protocol, + hostname: url.hostname, + port: url.port, + socketPath: isUnix && url.pathname, + path: options.path, + method: 'PUT', + headers: { + 'Content-Length': String(data.length), + 'Content-Type': 'application/msgpack', + 'Datadog-Container-ID': dockerId || '', + 'Datadog-Meta-Lang': 'nodejs', + 'Datadog-Meta-Lang-Version': process.version, + 'Datadog-Meta-Lang-Interpreter': process.jsEngine || 'v8', + 'Datadog-Meta-Tracer-Version': tracerVersion, + 'X-Datadog-Trace-Count': String(options.count) + }, + timeout + } + + const onResponse = res => { + let data = '' + + res.setTimeout(timeout) + res.on('data', chunk => { + data += chunk + }) + res.on('end', () => { + if (res.statusCode >= 200 && res.statusCode <= 299) { + try { + const response = data + done(null, response) + } catch (e) { + done(e) + } + } else { + const statusCode = res.statusCode + const statusText = http.STATUS_CODES[res.statusCode] + const error = new Error(`Error from the agent: ${statusCode} ${statusText}`) + + error.status = statusCode + + done(error, null) + } + }) + } + + const makeRequest = onError => { + const store = storage.getStore() + + storage.enterWith({ noop: true }) + + const req = client.request(httpOptions, onResponse) + + req.on('error', onError) + + req.setTimeout(timeout, req.abort) + req.write(data) + req.end() + + storage.enterWith(store) + } + + makeRequest(() => makeRequest(done)) // retry once on error + } +} + +module.exports = { Client } diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js new file mode 100644 index 00000000000..657e7c4bf53 --- /dev/null +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -0,0 +1,362 @@ +'use strict' + +const Chunk = require('../../../../packages/dd-trace/src/encode/chunk') +const { storage } = require('../../../../packages/datadog-core') +const { Client } = require('./client') + +const ARRAY_OF_TWO = 0x92 +const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB +const flushInterval = 2000 +const noop = () => {} +const eventTypes = { + KOA_REQUEST_START: 1, + ERROR: 2, + KOA_REQUEST_FINISH: 3 +} + +const float64Array = new Float64Array(1) +const uInt8Float64Array = new Uint8Array(float64Array.buffer) + +float64Array[0] = -1 + +const bigEndian = uInt8Float64Array[7] === 0 + +class Encoder { + constructor (limit = SOFT_LIMIT) { + this._limit = limit + this._eventBytes = new Chunk() + this._stringBytes = new Chunk() + this._client = new Client() + this._reset() + + process.once('beforeExit', () => this.flush()) + } + + count () { + return this._eventCount + } + + encodeKoaRequestStart (req) { + const bytes = this._eventBytes + const store = storage.getStore() + + if (!store || !store.traceContext) return + + // service name comes from process + // span name comes from event type + // resource comes from mixing http method and route + // error will be its own event + + this._encodeFixArray(bytes, 7) + this._encodeShort(bytes, eventTypes.KOA_REQUEST_START) // implied: name + this._encodeLong(bytes, 0) // figure out timestamp + this._encodeId(bytes, store.traceContext.traceId) + this._encodeId(bytes, store.traceContext.spanId) + this._encodeId(bytes, store.traceContext.parentId) // ??? + this._encodeString(bytes, req.method) + this._encodeString(bytes, req.url) + + this._afterEncode() + } + + encodeKoaRequestFinish (res) { + const bytes = this._eventBytes + const store = storage.getStore() + + if (!store || !store.traceContext) return + + this._encodeFixArray(bytes, 5) + this._encodeShort(bytes, eventTypes.KOA_REQUEST_FINISH) // implied: name + this._encodeLong(bytes, 0) // figure out timestamp + this._encodeId(bytes, store.traceContext.traceId) + this._encodeId(bytes, store.traceContext.spanId) + this._encodeShort(bytes, res.statusCode) + + this._afterEncode() + } + + encodeError (error) { + const bytes = this._eventBytes + const store = storage.getStore() + + if (!store || !store.traceContext) return // TODO: support errors without tracing + + this._encodeFixArray(bytes, 7) + this._encodeShort(bytes, eventTypes.ERROR) // implied: name + this._encodeLong(bytes, 0) // figure out timestamp + this._encodeId(bytes, store.traceContext.traceId) + this._encodeId(bytes, store.traceContext.spanId) + this._encodeString(bytes, error.name) + this._encodeString(bytes, error.message) + this._encodeString(bytes, error.stack) + + this._afterEncode() + } + + makePayload () { + const prefixSize = 1 + const stringSize = this._stringBytes.length + 5 + const eventSize = this._eventBytes.length + 5 + const buffer = Buffer.allocUnsafe(prefixSize + stringSize + eventSize) + + let offset = 0 + + buffer[offset++] = ARRAY_OF_TWO + + offset = this._writeStrings(buffer, offset) + offset = this._writeEvents(buffer, offset) + + this._reset() + + return buffer + } + + makePayload04 () { + const eventSize = this._eventBytes.length + 5 + const buffer = Buffer.allocUnsafe(eventSize) + + this._writeEvents(buffer) + + this._reset() + + return buffer + } + + flush (done = noop) { + const count = this.count() + + if (count === 0) return + + const data = this.makePayload() + const path = `/v1.0/events` + + this._timer = clearTimeout(this._timer) + this._client.request({ data, path, count }, done) + } + + reset () { + this._reset() + } + + _afterEncode () { + this._eventCount++ + + // we can go over the soft limit since the agent has a 50MB hard limit + if (this._eventBytes.length > this._limit || this._stringBytes.length > this._limit) { + this.flush() + } else if (!this._timer) { + this._timer = setTimeout(() => this.flush(), flushInterval).unref() + } + } + + _reset () { + this._eventCount = 0 + this._eventBytes.length = 0 + this._stringCount = 0 + this._stringBytes.length = 0 + this._stringMap = {} + + this._cacheString('') + } + + _encodeFixArray (bytes, size = 0) { + const offset = bytes.length + + bytes.reserve(1) + bytes.length += 1 + + bytes.buffer[offset] = 0x90 + size + } + + _encodeArrayPrefix (bytes, value) { + const length = value.length + const offset = bytes.length + + bytes.reserve(5) + bytes.length += 5 + + bytes.buffer[offset] = 0xdd + bytes.buffer[offset + 1] = length >> 24 + bytes.buffer[offset + 2] = length >> 16 + bytes.buffer[offset + 3] = length >> 8 + bytes.buffer[offset + 4] = length + } + + _encodeMapPrefix (bytes, keysLength) { + const offset = bytes.length + + bytes.reserve(5) + bytes.length += 5 + bytes.buffer[offset] = 0xdf + bytes.buffer[offset + 1] = keysLength >> 24 + bytes.buffer[offset + 2] = keysLength >> 16 + bytes.buffer[offset + 3] = keysLength >> 8 + bytes.buffer[offset + 4] = keysLength + } + + _encodeByte (bytes, value) { + bytes.reserve(1) + + bytes.buffer[bytes.length++] = value + } + + _encodeId (bytes, id) { + const offset = bytes.length + + bytes.reserve(9) + bytes.length += 9 + + id = id.toArray() + + bytes.buffer[offset] = 0xcf + bytes.buffer[offset + 1] = id[0] + bytes.buffer[offset + 2] = id[1] + bytes.buffer[offset + 3] = id[2] + bytes.buffer[offset + 4] = id[3] + bytes.buffer[offset + 5] = id[4] + bytes.buffer[offset + 6] = id[5] + bytes.buffer[offset + 7] = id[6] + bytes.buffer[offset + 8] = id[7] + } + + _encodeInteger (bytes, value) { + const offset = bytes.length + + bytes.reserve(5) + bytes.length += 5 + + bytes.buffer[offset] = 0xce + bytes.buffer[offset + 1] = value >> 24 + bytes.buffer[offset + 2] = value >> 16 + bytes.buffer[offset + 3] = value >> 8 + bytes.buffer[offset + 4] = value + } + + _encodeShort (bytes, value) { + const offset = bytes.length + + bytes.reserve(3) + bytes.length += 3 + + bytes.buffer[offset] = 0xcd + bytes.buffer[offset + 1] = value >> 8 + bytes.buffer[offset + 2] = value + } + + _encodeLong (bytes, value) { + const offset = bytes.length + const hi = (value / Math.pow(2, 32)) >> 0 + const lo = value >>> 0 + + bytes.reserve(9) + bytes.length += 9 + + bytes.buffer[offset] = 0xcf + bytes.buffer[offset + 1] = hi >> 24 + bytes.buffer[offset + 2] = hi >> 16 + bytes.buffer[offset + 3] = hi >> 8 + bytes.buffer[offset + 4] = hi + bytes.buffer[offset + 5] = lo >> 24 + bytes.buffer[offset + 6] = lo >> 16 + bytes.buffer[offset + 7] = lo >> 8 + bytes.buffer[offset + 8] = lo + } + + _encodeMap (bytes, value) { + const keys = Object.keys(value) + const validKeys = keys.filter(key => typeof value[key] === 'string' || typeof value[key] === 'number') + + this._encodeMapPrefix(bytes, validKeys.length) + + for (const key of validKeys) { + this._encodeString(bytes, key) + this._encodeValue(bytes, value[key]) + } + } + + _encodeValue (bytes, value) { + switch (typeof value) { + case 'string': + this._encodeString(bytes, value) + break + case 'number': + this._encodeFloat(bytes, value) + break + default: + // should not happen + } + } + + _encodeString (bytes, value = '') { + this._cacheString(value) + this._encodeInteger(bytes, this._stringMap[value]) + } + + _encodeString04 (bytes, value = '') { + this._cacheString(value) + + const { start, end } = this._stringMap[value] + + this._stringBytes.copy(bytes, start, end) + } + + _encodeFloat (bytes, value) { + float64Array[0] = value + + const offset = bytes.length + bytes.reserve(9) + bytes.length += 9 + + bytes.buffer[offset] = 0xcb + + if (bigEndian) { + for (let i = 0; i <= 7; i++) { + bytes.buffer[offset + i + 1] = uInt8Float64Array[i] + } + } else { + for (let i = 7; i >= 0; i--) { + bytes.buffer[bytes.length - i - 1] = uInt8Float64Array[i] + } + } + } + + _cacheString (value) { + if (!(value in this._stringMap)) { + this._stringMap[value] = this._stringCount++ + this._stringBytes.write(value) + } + } + + _cacheString04 (value) { + if (!(value in this._stringMap)) { + this._stringCount++ + this._stringMap[value] = { + start: this._stringBytes.length, + end: this._stringBytes.length + this._stringBytes.write(value) + } + } + } + + _writeArrayPrefix (buffer, offset, count) { + buffer[offset++] = 0xdd + buffer.writeUInt32BE(count, offset) + + return offset + 4 + } + + _writeStrings (buffer, offset) { + offset = this._writeArrayPrefix(buffer, offset, this._stringCount) + offset += this._stringBytes.buffer.copy(buffer, offset, 0, this._stringBytes.length) + + return offset + } + + _writeEvents (buffer, offset = 0) { + offset = this._writeArrayPrefix(buffer, offset, this._eventCount) + offset += this._eventBytes.buffer.copy(buffer, offset, 0, this._eventBytes.length) + + return offset + } +} + +module.exports = { Encoder } diff --git a/benchmark/sirun/plugin-koa/internal-tracer/id.js b/benchmark/sirun/plugin-koa/internal-tracer/id.js new file mode 100644 index 00000000000..5873f5fa5d5 --- /dev/null +++ b/benchmark/sirun/plugin-koa/internal-tracer/id.js @@ -0,0 +1,126 @@ +'use strict' + +const { randomFillSync } = require('crypto') + +const UINT_MAX = 4294967296 + +const data = new Uint8Array(8 * 8192) +const zeroIdBuffer = new Uint8Array(8) +const zeroId = { + toArray: () => zeroIdBuffer, + toString: () => '0', + toJSON: () => '0' +} + +let batch = 0 + +function id (value, raddix) { + const buffer = value + ? fromNumberString(value, raddix) + : pseudoRandom() + + return { + toArray: () => buffer, + toString: (raddix) => toNumberString(buffer, raddix), + toJSON: () => toNumberString(buffer) + } +} + +function pseudoRandom () { + if (batch === 0) { + randomFillSync(data) + } + + batch = (batch + 1) % 8192 + + const offset = batch * 8 + + return [ + data[offset] & 0x7F, // only positive int64 + data[offset + 1], + data[offset + 2], + data[offset + 3], + data[offset + 4], + data[offset + 5], + data[offset + 6], + data[offset + 7] + ] +} + +function fromNumberString (str, raddix = 10) { + const len = str.length + + let pos = 0 + let high = 0 + let low = 0 + + if (str[0] === '-') pos++ + + const sign = pos + + while (pos < len) { + const chr = parseInt(str[pos++], raddix) + + if (!(chr >= 0)) break // NaN + + low = low * raddix + chr + high = high * raddix + Math.floor(low / UINT_MAX) + low %= UINT_MAX + } + + if (sign) { + high = ~high + + if (low) { + low = UINT_MAX - low + } else { + high++ + } + } + + if (high === 0 && low === 0) return zeroId + + const buffer = new Array(8) // TODO: use existing buffer + + writeUInt32BE(buffer, high, 0) + writeUInt32BE(buffer, low, 4) + + return buffer +} + +function toNumberString (buffer, radix = 10) { + let high = readInt32(buffer, 0) + let low = readInt32(buffer, 4) + let str = '' + + while (1) { + const mod = (high % radix) * UINT_MAX + low + + high = Math.floor(high / radix) + low = Math.floor(mod / radix) + str = (mod % radix).toString(radix) + str + + if (!high && !low) break + } + + return str +} + +function readInt32 (buffer, offset) { + return (buffer[offset + 0] * 16777216) + + (buffer[offset + 1] << 16) + + (buffer[offset + 2] << 8) + + buffer[offset + 3] +} + +function writeUInt32BE (buffer, value, offset) { + buffer[3 + offset] = value & 255 + value = value >> 8 + buffer[2 + offset] = value & 255 + value = value >> 8 + buffer[1 + offset] = value & 255 + value = value >> 8 + buffer[0 + offset] = value & 255 +} + +module.exports = { id, zeroId } diff --git a/benchmark/sirun/plugin-koa/internal-tracer/index.js b/benchmark/sirun/plugin-koa/internal-tracer/index.js new file mode 100644 index 00000000000..aac26892504 --- /dev/null +++ b/benchmark/sirun/plugin-koa/internal-tracer/index.js @@ -0,0 +1,5 @@ +'use strict' + +require('./patch') +require('./storage') +require('./trace') diff --git a/benchmark/sirun/plugin-koa/internal-tracer/patch.js b/benchmark/sirun/plugin-koa/internal-tracer/patch.js new file mode 100644 index 00000000000..bb9ff542ca4 --- /dev/null +++ b/benchmark/sirun/plugin-koa/internal-tracer/patch.js @@ -0,0 +1,32 @@ +'use strict' + +const { channel } = require('diagnostics_channel') +const Hook = require('../../../../packages/dd-trace/src/ritm') + +const startChannel = channel('apm:koa:request:start') +const endChannel = channel('apm:koa:request:end') +const errorChannel = channel('apm:koa:request:error') +const asyncEndChannel = channel('apm:koa:request:async-end') + +Hook(['koa'], function (Koa, name, basedir) { + if (name !== 'koa/lib/application.js') return Koa + + const { handleRequest } = Koa.prototype + + Koa.prototype.handleRequest = function (ctx, fnMiddleware) { + startChannel.publish(ctx) + + const promise = handleRequest.apply(this, arguments) + .then(() => asyncEndChannel.publish(ctx), error => { + errorChannel.publish(error) + asyncEndChannel.publish(ctx) + throw error + }) + + endChannel.publish(ctx) + + return promise + } + + return Koa +}) diff --git a/benchmark/sirun/plugin-koa/internal-tracer/storage.js b/benchmark/sirun/plugin-koa/internal-tracer/storage.js new file mode 100644 index 00000000000..5416804df2c --- /dev/null +++ b/benchmark/sirun/plugin-koa/internal-tracer/storage.js @@ -0,0 +1,20 @@ +'use strict' + +const { channel } = require('diagnostics_channel') +const { storage } = require('../../../../packages/datadog-core') + +const startChannel = channel('apm:koa:request:start') +const endChannel = channel('apm:koa:request:end') + +const stores = [] + +startChannel.subscribe(() => { + const store = storage.getStore() + + stores.push(store) + storage.enterWith({ ...store }) +}) + +endChannel.subscribe(() => { + storage.enterWith(stores.pop()) +}) diff --git a/benchmark/sirun/plugin-koa/internal-tracer/trace.js b/benchmark/sirun/plugin-koa/internal-tracer/trace.js new file mode 100644 index 00000000000..43ed0259de2 --- /dev/null +++ b/benchmark/sirun/plugin-koa/internal-tracer/trace.js @@ -0,0 +1,45 @@ +'use strict' + +const { channel } = require('diagnostics_channel') +const { Encoder } = require('./encoder') +const { storage } = require('../../../../packages/datadog-core') +const { id, zeroId } = require('./id') + +const startChannel = channel('apm:koa:request:start') +const errorChannel = channel('apm:koa:request:error') +const asyncEndChannel = channel('apm:koa:request:async-end') + +const encoder = new Encoder() + +class TraceContext { + constructor (childOf) { + if (childOf) { + this.traceId = childOf.traceId + this.spanId = id() + this.parentId = childOf.spanId + } else { + this.traceId = id() + this.spanId = this.traceId + this.parentId = zeroId + } + } +} + +startChannel.subscribe(({ req }) => { + const store = storage.getStore() + const traceContext = new TraceContext(store.traceContext) + + store.traceContext = traceContext + + encoder.encodeKoaRequestStart(req) +}) + +errorChannel.subscribe(error => { + encoder.encodeError(error) +}) + +asyncEndChannel.subscribe(res => { + encoder.encodeKoaRequestFinish(res) + + // TODO: restore parent context +}) diff --git a/benchmark/sirun/plugin-koa/meta.json b/benchmark/sirun/plugin-koa/meta.json new file mode 100644 index 00000000000..86f7035f9bf --- /dev/null +++ b/benchmark/sirun/plugin-koa/meta.json @@ -0,0 +1,26 @@ +{ + "name": "koa", + "setup": "node setup", + "run": "node server", + "env": { + "PORT": "3030", + "REQUESTS": "100000" + }, + "iterations": 10, + "cachegrind": false, + "variants": { + "baseline": {}, + "with-internal-tracer": { + "env": { + "WITH_INTERNAL_TRACER": "true", + "DD_TRACE_AGENT_PROTOCOL_VERSION": "0.5" + } + }, + "with-tracer": { + "env": { + "WITH_TRACER": "true", + "DD_TRACE_AGENT_PROTOCOL_VERSION": "0.5" + } + } + } +} diff --git a/benchmark/sirun/plugin-koa/server.js b/benchmark/sirun/plugin-koa/server.js new file mode 100644 index 00000000000..023119a0943 --- /dev/null +++ b/benchmark/sirun/plugin-koa/server.js @@ -0,0 +1,44 @@ +'use strict' + +const { PORT, REQUESTS, WITH_INTERNAL_TRACER, WITH_TRACER } = process.env + +if (WITH_TRACER === 'true') { + const tracer = require('../../..') + tracer.init({ + startupLogs: false, + plugins: false + }) + tracer.use('http', { enabled: true }) + tracer.use('koa', { enabled: true, middleware: true }) +} + +if (WITH_INTERNAL_TRACER === 'true') { + require('./internal-tracer') +} + +const http = require('http') +const Koa = require('../../../versions/koa/node_modules/koa') +const net = require('net') +const app = new Koa() + +const port = parseInt(PORT) +const requests = parseInt(REQUESTS) + +let readyServer +let total = 0 + +app.use(ctx => { + ctx.body = 'OK' + + if (++total === requests) { + server.close() + readyServer.close() + } +}) + +const server = http.createServer(app.callback()) + +server.listen(port, () => { + readyServer = net.createServer(() => {}) + readyServer.listen(port + 1) +}) diff --git a/benchmark/sirun/plugin-koa/setup.js b/benchmark/sirun/plugin-koa/setup.js new file mode 100644 index 00000000000..cd09f1d170c --- /dev/null +++ b/benchmark/sirun/plugin-koa/setup.js @@ -0,0 +1,13 @@ +'use strict' + +const { spawn } = require('child_process') + +const port = parseInt(process.env.PORT) +const requests = parseInt(process.env.REQUESTS) +const options = { + detached: true, + stdio: 'ignore', + shell: true +} + +spawn(`npx wait-on tcp:${port + 1} && npx autocannon -a ${requests} http://localhost:${port}/hello`, options).unref() From 326e5f9200be4ab9e64777640129954ea8470d13 Mon Sep 17 00:00:00 2001 From: rochdev Date: Mon, 20 Feb 2023 21:22:09 -0500 Subject: [PATCH 02/19] add collector to convert events to spans --- .editorconfig | 3 + .../plugin-koa/internal-tracer/client.js | 2 +- .../plugin-koa/internal-tracer/encoder.js | 41 +- .../sirun/plugin-koa/internal-tracer/trace.js | 2 +- collector/.gitignore | 1 + collector/Cargo.lock | 1259 +++++++++++++++++ collector/Cargo.toml | 15 + collector/bench/single.js | 37 + collector/src/main.rs | 391 +++++ 9 files changed, 1737 insertions(+), 14 deletions(-) create mode 100644 collector/.gitignore create mode 100644 collector/Cargo.lock create mode 100644 collector/Cargo.toml create mode 100644 collector/bench/single.js create mode 100644 collector/src/main.rs diff --git a/.editorconfig b/.editorconfig index 7fdb734b2ee..2749b2502c2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,5 +10,8 @@ charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true +[*.rs] +indent_size = 4 + [*.md] trim_trailing_whitespace = false diff --git a/benchmark/sirun/plugin-koa/internal-tracer/client.js b/benchmark/sirun/plugin-koa/internal-tracer/client.js index 3b172a015ae..bea92069bbc 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/client.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/client.js @@ -14,7 +14,7 @@ class Client { request (options, done) { if (options.count === 0) return - const url = new URL(DD_TRACE_AGENT_URL || 'http://127.0.0.1:8126') + const url = new URL(DD_TRACE_AGENT_URL || 'http://127.0.0.1:8127') const isSecure = url.protocol === 'https:' const isUnix = url.protocol === 'unix:' const client = isSecure ? https : http diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js index 657e7c4bf53..c008ba56251 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -4,6 +4,10 @@ const Chunk = require('../../../../packages/dd-trace/src/encode/chunk') const { storage } = require('../../../../packages/datadog-core') const { Client } = require('./client') +const processStartTime = BigInt(Date.now() * 1e6) +const processStartTicks = process.hrtime.bigint() +const now = () => Number(processStartTime + process.hrtime.bigint() - processStartTicks) +const service = process.env.DD_SERVICE || 'unnamed-node-app' const ARRAY_OF_TWO = 0x92 const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB const flushInterval = 2000 @@ -49,7 +53,7 @@ class Encoder { this._encodeFixArray(bytes, 7) this._encodeShort(bytes, eventTypes.KOA_REQUEST_START) // implied: name - this._encodeLong(bytes, 0) // figure out timestamp + this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) this._encodeId(bytes, store.traceContext.parentId) // ??? @@ -67,7 +71,7 @@ class Encoder { this._encodeFixArray(bytes, 5) this._encodeShort(bytes, eventTypes.KOA_REQUEST_FINISH) // implied: name - this._encodeLong(bytes, 0) // figure out timestamp + this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) this._encodeShort(bytes, res.statusCode) @@ -83,7 +87,7 @@ class Encoder { this._encodeFixArray(bytes, 7) this._encodeShort(bytes, eventTypes.ERROR) // implied: name - this._encodeLong(bytes, 0) // figure out timestamp + this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) this._encodeString(bytes, error.name) @@ -93,7 +97,8 @@ class Encoder { this._afterEncode() } - makePayload () { + // TODO: support new payload format + makePayload05 () { const prefixSize = 1 const stringSize = this._stringBytes.length + 5 const eventSize = this._eventBytes.length + 5 @@ -111,11 +116,23 @@ class Encoder { return buffer } - makePayload04 () { + makePayload () { const eventSize = this._eventBytes.length + 5 - const buffer = Buffer.allocUnsafe(eventSize) + const serviceLength = Buffer.byteLength(service) + const buffer = Buffer.allocUnsafe(eventSize + 18 + serviceLength) + + let offset = 0 + + buffer[offset++] = 0x82 // fixmap(2) - this._writeEvents(buffer) + buffer[offset++] = 0xa7 // fixstr(7) + offset += buffer.write('service', offset) + buffer[offset++] = 0xd9 // str8 + buffer[offset++] = serviceLength + offset += buffer.write(service, offset) + buffer[offset++] = 0xa6 // fixstr(6) + offset += buffer.write('events', offset) + offset = this._writeEvents(buffer, offset) this._reset() @@ -128,7 +145,7 @@ class Encoder { if (count === 0) return const data = this.makePayload() - const path = `/v1.0/events` + const path = `/v0.1/events` this._timer = clearTimeout(this._timer) this._client.request({ data, path, count }, done) @@ -287,12 +304,12 @@ class Encoder { } } - _encodeString (bytes, value = '') { + _encodeString05 (bytes, value = '') { this._cacheString(value) this._encodeInteger(bytes, this._stringMap[value]) } - _encodeString04 (bytes, value = '') { + _encodeString (bytes, value = '') { this._cacheString(value) const { start, end } = this._stringMap[value] @@ -320,14 +337,14 @@ class Encoder { } } - _cacheString (value) { + _cacheString05 (value) { if (!(value in this._stringMap)) { this._stringMap[value] = this._stringCount++ this._stringBytes.write(value) } } - _cacheString04 (value) { + _cacheString (value) { if (!(value in this._stringMap)) { this._stringCount++ this._stringMap[value] = { diff --git a/benchmark/sirun/plugin-koa/internal-tracer/trace.js b/benchmark/sirun/plugin-koa/internal-tracer/trace.js index 43ed0259de2..85d7950225f 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/trace.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/trace.js @@ -38,7 +38,7 @@ errorChannel.subscribe(error => { encoder.encodeError(error) }) -asyncEndChannel.subscribe(res => { +asyncEndChannel.subscribe(({ res }) => { encoder.encodeKoaRequestFinish(res) // TODO: restore parent context diff --git a/collector/.gitignore b/collector/.gitignore new file mode 100644 index 00000000000..eb5a316cbd1 --- /dev/null +++ b/collector/.gitignore @@ -0,0 +1 @@ +target diff --git a/collector/Cargo.lock b/collector/Cargo.lock new file mode 100644 index 00000000000..c42b674b602 --- /dev/null +++ b/collector/Cargo.lock @@ -0,0 +1,1259 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "async-trait" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "axum" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fb79c228270dcf2426e74864cabc94babb5dbab01a4314e702d2f16540e1591" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "dd-trace-collector" +version = "0.1.0" +dependencies = [ + "axum", + "reqwest", + "rmp", + "rmp-serde", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" + +[[package]] +name = "futures-sink" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" + +[[package]] +name = "futures-task" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" + +[[package]] +name = "futures-util" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "h2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" + +[[package]] +name = "itoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.45.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "openssl" +version = "0.10.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.45.0", +] + +[[package]] +name = "paste" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "proc-macro2" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rmp" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5b13be192e0220b8afb7222aa5813cb62cc269ebb5cac346ca6487681d2913e" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + +[[package]] +name = "rustversion" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" + +[[package]] +name = "ryu" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" + +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "syn" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.42.0", +] + +[[package]] +name = "tokio-macros" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "unicode-bidi" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" + +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/collector/Cargo.toml b/collector/Cargo.toml new file mode 100644 index 00000000000..55edbf4fa58 --- /dev/null +++ b/collector/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "dd-trace-collector" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axum = "0.6.7" +reqwest = "0.11.14" +rmp = "0.8.11" +rmp-serde = "1.1.1" +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.93" +tokio = { version = "1.25.0", features = ["full"] } diff --git a/collector/bench/single.js b/collector/bench/single.js new file mode 100644 index 00000000000..58216fc865d --- /dev/null +++ b/collector/bench/single.js @@ -0,0 +1,37 @@ +const http = require('http') +const id = require('../../packages/dd-trace/src/id') +const msgpack = require('msgpack-lite') +const codec = msgpack.createCodec({ int64: true }) + +const traceId = id() +const spanId = id() +const startTime = id((Date.now() * 1e6).toString(), 10) +const finishTime = id(((Date.now() + 100) * 1e6).toString(), 10) + +const payload = msgpack.encode({ + 'events': [ + [1, startTime, traceId, spanId, 0, 'GET', '/some/path'], + [3, finishTime, traceId, spanId, 200] + ] +}, { codec }) + +const req = http.request({ + method: 'put', + port: 8127, + path: '/v0.1/events' +}, res => { + let data = '' + + res.on('data', chunk => { + data += chunk + }) + + res.on('end', () => { + console.log(`Response: ${data}`) // eslint-disable-line no-console + }) +}) + +req.setHeader('Content-Type', 'application/msgpack') +req.write(payload) + +req.end() diff --git a/collector/src/main.rs b/collector/src/main.rs new file mode 100644 index 00000000000..3f8ba6e6031 --- /dev/null +++ b/collector/src/main.rs @@ -0,0 +1,391 @@ +use axum::body::Bytes; +use axum::extract::{DefaultBodyLimit,State}; +use axum::routing::put; +use axum::Router; +use rmp::encode; +use rmp::encode::ByteBuf; +use serde_json::Value; +use tokio::sync::mpsc; +use tokio::sync::mpsc::Sender; +use tokio::sync::mpsc::Receiver; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Arc; +use serde::Deserialize; + +type Traces = HashMap; + +#[derive(Debug)] +struct Span { + span_type: Option, + trace_id: u64, + span_id: u64, + parent_id: u64, + name: String, + resource: String, + service: String, + error: u64, + start: u64, + duration: u64, + meta: HashMap, + metrics: HashMap +} + +#[derive(Debug)] +struct Trace { + started: u64, + finished: u64, + spans: HashMap +} + +#[derive(Deserialize, Debug)] +struct Payload { + service: String, + events: Value +} + +struct AppState { + tx: Sender +} + +// TODO: Use hyper directly instead of axum/reqwest. +// TODO: Decouple processing from transport. +// TODO: Store traces per connection and cleanup on connection close. +// TODO: Read MsgPack manually and copy bytes to span buffer directly. +// TODO: Add support for payload metadata (i.e. service, language). +// TODO: Use string table. +// TODO: Read events into structs instead of serde values. +// TODO: Event for adding trace tags. +// TODO: Event for adding baggage items. +// TODO: Add support for sampling. +// TODO: Support sending traces directly to Datadog. +// TODO: Optimize to minimize allocations and copies. +// TODO: Split in modules. +// TODO: Add tests. +// TODO: Add benchmarks. + +#[tokio::main] +async fn main() { + let (tx, mut rx): (Sender, Receiver) = mpsc::channel(100); + + let processor = tokio::spawn(async move { + let mut traces = Traces::new(); + + while let Some(payload) = rx.recv().await { + let events = payload.events.as_array().unwrap().to_vec(); + + for event in events { + process_event(&mut traces, event, &payload); + } + + flush(&mut traces).await; + } + }); + + let state = Arc::new(AppState { tx }); + let addr = SocketAddr::from(([127, 0, 0, 1], 8127)); + let app = Router::new() + .route("/v0.1/events", put(handle_events)) + .layer(DefaultBodyLimit::max(1024 * 1024 * 16)) // 16MB + .with_state(state); + + let server = axum::Server::bind(&addr) + .serve(app.into_make_service()); + + let (_, _) = tokio::join!(server, processor); +} + +async fn handle_events(State(state): State>, bytes: Bytes) { + let data: Vec = bytes.try_into().unwrap(); + let payload: Payload = rmp_serde::from_slice(&data).unwrap(); + + // println!("{:#?}", payload); + + state.tx.send(payload).await.unwrap(); +} + +fn process_event(traces: &mut Traces, event: Value, payload: &Payload) { + let event_type = event.get(0).unwrap().as_u64().unwrap(); + + match event_type { + 1 => process_start_koa_request(traces, &event, payload), + 2 => process_add_error(traces, &event), + 3 => process_finish_koa_request(traces, &event), + 4 => process_start_span(traces, &event, payload), + 5 => process_finish_span(traces, &event), + 6 => process_add_tags(traces, &event), + _ => () + } +} + +fn process_add_error(traces: &mut Traces, event: &Value) { + let maybe_trace = traces.get_mut(&event[2].as_u64().unwrap()); + + match maybe_trace { + Some(trace) => { + let maybe_span = trace.spans.get_mut(&event[3].as_u64().unwrap()); + + match maybe_span { + Some(mut span) => { + span.error = 1; + span.meta.insert(String::from("error.type"), event[4].as_str().unwrap().to_string()); + span.meta.insert(String::from("error.message"), event[5].as_str().unwrap().to_string()); + span.meta.insert(String::from("error.stack"), event[6].as_str().unwrap().to_string()); + }, + None => () + } + }, + None => () + } +} + +fn process_add_tags(traces: &mut Traces, event: &Value) { + let maybe_trace = traces.get_mut(&event[2].as_u64().unwrap()); + + match maybe_trace { + Some(trace) => { + let maybe_span = trace.spans.get_mut(&event[3].as_u64().unwrap()); + + match maybe_span { + Some(span) => { + add_tags_from_value(span, &event[4]); + }, + None => () + } + }, + None => () + } +} + +fn process_start_span(traces: &mut Traces, event: &Value, payload: &Payload) { + let mut meta = HashMap::new(); + let mut metrics = HashMap::new(); + + for (k, v) in event[9].as_object().unwrap() { + match v { + Value::Number(v) => { + metrics.insert(k.to_string(), v.as_f64().unwrap()); + }, + Value::String(v) => { + meta.insert(k.to_string(), v.to_string()); + } + _ => () + } + } + + let span_type = event[5].as_str().unwrap(); + let service = event[8].as_str().unwrap(); + let mut span = Span { + span_type: if span_type.is_empty() { None } else { Some(span_type.to_string()) }, + trace_id: event[2].as_u64().unwrap(), + span_id: event[3].as_u64().unwrap(), + parent_id: event[4].as_u64().unwrap(), + name: event[6].as_str().unwrap().to_string(), + resource: event[7].as_str().unwrap().to_string(), + service: if service.is_empty() { payload.service.to_string() } else { service.to_string() }, + error: 0, + start: event[1].as_u64().unwrap(), + duration: 0, + meta, + metrics + }; + + add_tags_from_value(&mut span, &event[9]); + + start_span(traces, span); +} + +fn process_finish_span(traces: &mut Traces, event: &Value) { + let maybe_trace = traces.get_mut(&event[2].as_u64().unwrap()); + + match maybe_trace { + Some(mut trace) => { + let maybe_span = trace.spans.get_mut(&event[3].as_u64().unwrap()); + + match maybe_span { + Some(mut span) => { + trace.finished += 1; + + span.duration = event[1].as_u64().unwrap() - span.start; + + add_tags_from_value(span, &event[4]); + }, + None => () + } + }, + None => () + } +} + +fn process_start_koa_request(traces: &mut Traces, event: &Value, payload: &Payload) { + let mut meta = HashMap::new(); + let metrics = HashMap::new(); + + let method = event[5].as_str().unwrap().to_string(); + let url = event[6].as_str().unwrap().to_string(); // TODO: route not url + let resource = format!("{method} {url}"); + + meta.insert(String::from("http.method"), method); + meta.insert(String::from("http.url"), url); + + let span = Span { + span_type: Some(String::from("web")), + trace_id: event[2].as_u64().unwrap(), + span_id: event[3].as_u64().unwrap(), + parent_id: event[4].as_u64().unwrap(), + name: String::from("koa.request"), + resource, + service: payload.service.to_string(), + error: 0, + start: event[1].as_u64().unwrap(), + duration: 0, + meta, + metrics + }; + + start_span(traces, span); +} + +fn process_finish_koa_request(traces: &mut Traces, event: &Value) { + let maybe_trace = traces.get_mut(&event[2].as_u64().unwrap()); + + match maybe_trace { + Some(mut trace) => { + let maybe_span = trace.spans.get_mut(&event[3].as_u64().unwrap()); + + match maybe_span { + Some(mut span) => { + trace.finished += 1; + + span.duration = event[1].as_u64().unwrap() - span.start; + span.meta.insert(String::from("http.status_code"), event[4].as_u64().unwrap().to_string()); + }, + None => () + } + }, + None => () + } +} + +fn start_span(traces: &mut Traces, span: Span) { + let trace = traces.entry(span.trace_id).or_insert(Trace { + started: 0, + finished: 0, + spans: HashMap::new() + }); + + trace.started += 1; + trace.spans.insert(span.span_id, span); +} + +fn add_tags_from_value(span: &mut Span, value: &Value) { + for (k, v) in value.as_object().unwrap() { + match v { + Value::Number(v) => { + span.metrics.insert(k.to_string(), v.as_f64().unwrap()); + }, + Value::String(v) => { + span.meta.insert(k.to_string(), v.to_string()); + } + _ => () + } + } +} + +async fn flush(traces: &mut Traces) { + let mut wr = ByteBuf::new(); + let finished_traces: Vec<&Trace> = traces.values().filter(|t| t.started == t.finished).collect(); + let trace_count = finished_traces.len(); + + if trace_count > 0 { + encode_traces(&mut wr, finished_traces); + + traces.retain(|_, t| t.started != t.finished); + + let client = reqwest::Client::new(); + let data: Vec = wr.as_vec().to_vec(); + + // println!("{:#?}", data); + + client.put("http://localhost:8126/v0.4/traces") + .header("Content-Type", "application/msgpack") + .header("X-Datadog-Trace-Count", trace_count.to_string()) + // .header("Datadog-Meta-Tracer-Version", "") + // .header("Datadog-Meta-Lang", "") + // .header("Datadog-Meta-Lang-Version", "") + // .header("Datadog-Meta-Lang-Interpreter", "") + .body(data) + .send() + .await + .unwrap(); + } +} + +fn encode_traces(wr: &mut ByteBuf, traces: Vec<&Trace>) { + encode::write_array_len(wr, traces.len() as u32).unwrap(); + + for trace in traces { + encode_trace(wr, trace); + } +} + +fn encode_trace(wr: &mut ByteBuf, trace: &Trace) { + encode::write_array_len(wr, trace.spans.len() as u32).unwrap(); + + for span in trace.spans.values() { + // println!("{:#?}", test); + + match &span.span_type { + Some(span_type) => { + encode::write_map_len(wr, 12).unwrap(); + encode::write_str(wr, "type").unwrap(); + encode::write_str(wr, span_type.as_str()).unwrap(); + }, + None => { + encode::write_map_len(wr, 11).unwrap(); + } + } + + encode::write_str(wr, "trace_id").unwrap(); + encode::write_uint(wr, span.trace_id).unwrap(); + encode::write_str(wr, "span_id").unwrap(); + encode::write_uint(wr, span.span_id).unwrap(); + encode::write_str(wr, "parent_id").unwrap(); + encode::write_uint(wr, span.parent_id).unwrap(); + encode::write_str(wr, "name").unwrap(); + encode::write_str(wr, span.name.as_str()).unwrap(); + encode::write_str(wr, "resource").unwrap(); + encode::write_str(wr, span.resource.as_str()).unwrap(); + encode::write_str(wr, "service").unwrap(); + encode::write_str(wr, span.service.as_str()).unwrap(); + encode::write_str(wr, "error").unwrap(); + encode::write_uint(wr, span.error).unwrap(); + encode::write_str(wr, "start").unwrap(); + encode::write_uint(wr, span.start).unwrap(); + encode::write_str(wr, "duration").unwrap(); + encode::write_uint(wr, span.duration + 1).unwrap(); + + encode_meta(wr, &span.meta); + encode_metrics(wr, &span.metrics); + } +} + +fn encode_meta(wr: &mut ByteBuf, map: &HashMap) { + encode::write_str(wr, "meta").unwrap(); + encode::write_map_len(wr, map.len() as u32).unwrap(); + + for (k, v) in map { + encode::write_str(wr, k.as_str()).unwrap(); + encode::write_str(wr, v.as_str()).unwrap(); + } +} + +fn encode_metrics(wr: &mut ByteBuf, map: &HashMap) { + encode::write_str(wr, "metrics").unwrap(); + encode::write_map_len(wr, map.len() as u32).unwrap(); + + for (k, v) in map { + encode::write_str(wr, k.as_str()).unwrap(); + encode::write_f64(wr, *v).unwrap(); + } +} From d91919c91963895779f1761e4f85732fe3de394e Mon Sep 17 00:00:00 2001 From: rochdev Date: Tue, 21 Feb 2023 21:09:51 -0500 Subject: [PATCH 03/19] remove axum and reqwest dependencies and add payload metadata --- .../plugin-koa/internal-tracer/encoder.js | 20 +- collector/Cargo.lock | 619 +----------------- collector/Cargo.toml | 3 +- collector/src/main.rs | 134 ++-- 4 files changed, 88 insertions(+), 688 deletions(-) diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js index c008ba56251..2f2dbb7cabe 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -28,6 +28,7 @@ const bigEndian = uInt8Float64Array[7] === 0 class Encoder { constructor (limit = SOFT_LIMIT) { this._limit = limit + this._metadataBytes = new Chunk(1024) this._eventBytes = new Chunk() this._stringBytes = new Chunk() this._client = new Client() @@ -117,19 +118,21 @@ class Encoder { } makePayload () { - const eventSize = this._eventBytes.length + 5 - const serviceLength = Buffer.byteLength(service) - const buffer = Buffer.allocUnsafe(eventSize + 18 + serviceLength) + this._encodeMap(this._metadataBytes, { service }) // HACK + + const metadataBytes = this._metadataBytes + const metadataSize = metadataBytes.length + 9 + const eventSize = this._eventBytes.length + 5 + 7 + const buffer = Buffer.allocUnsafe(1 + metadataSize + eventSize) let offset = 0 buffer[offset++] = 0x82 // fixmap(2) - buffer[offset++] = 0xa7 // fixstr(7) - offset += buffer.write('service', offset) - buffer[offset++] = 0xd9 // str8 - buffer[offset++] = serviceLength - offset += buffer.write(service, offset) + buffer[offset++] = 0xa8 // fixstr(8) + offset += buffer.write('metadata', offset) + offset += metadataBytes.buffer.copy(buffer, offset, 0, metadataBytes.length) + buffer[offset++] = 0xa6 // fixstr(6) offset += buffer.write('events', offset) offset = this._writeEvents(buffer, offset) @@ -167,6 +170,7 @@ class Encoder { } _reset () { + this._metadataBytes.length = 0 this._eventCount = 0 this._eventBytes.length = 0 this._stringCount = 0 diff --git a/collector/Cargo.lock b/collector/Cargo.lock index c42b674b602..b33565841f5 100644 --- a/collector/Cargo.lock +++ b/collector/Cargo.lock @@ -2,91 +2,18 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "async-trait" -version = "0.1.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "axum" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fb79c228270dcf2426e74864cabc94babb5dbab01a4314e702d2f16540e1591" -dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower", - "tower-http", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "base64" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" - [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bumpalo" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" - [[package]] name = "byteorder" version = "1.4.3" @@ -99,40 +26,17 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" -[[package]] -name = "cc" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" - [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "core-foundation" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" - [[package]] name = "dd-trace-collector" version = "0.1.0" dependencies = [ - "axum", - "reqwest", + "hyper", "rmp", "rmp-serde", "serde", @@ -140,54 +44,12 @@ dependencies = [ "tokio", ] -[[package]] -name = "encoding_rs" -version = "0.8.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" -dependencies = [ - "percent-encoding", -] - [[package]] name = "futures-channel" version = "0.3.26" @@ -283,12 +145,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-range-header" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" - [[package]] name = "httparse" version = "1.8.0" @@ -325,29 +181,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "idna" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "indexmap" version = "1.9.2" @@ -358,42 +191,12 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "ipnet" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" - [[package]] name = "itoa" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" -[[package]] -name = "js-sys" -version = "0.3.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - [[package]] name = "libc" version = "0.2.139" @@ -419,24 +222,12 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "matchit" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" - [[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" -[[package]] -name = "mime" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" - [[package]] name = "mio" version = "0.8.6" @@ -449,24 +240,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "num-traits" version = "0.2.15" @@ -492,51 +265,6 @@ version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" -[[package]] -name = "openssl" -version = "0.10.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -566,32 +294,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" -[[package]] -name = "percent-encoding" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" - -[[package]] -name = "pin-project" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.9" @@ -604,12 +306,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - [[package]] name = "proc-macro2" version = "1.0.51" @@ -637,52 +333,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - -[[package]] -name = "reqwest" -version = "0.11.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - [[package]] name = "rmp" version = "0.8.11" @@ -705,56 +355,18 @@ dependencies = [ "serde", ] -[[package]] -name = "rustversion" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" - [[package]] name = "ryu" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" -[[package]] -name = "schannel" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" -dependencies = [ - "windows-sys 0.42.0", -] - [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "security-framework" -version = "2.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "serde" version = "1.0.152" @@ -786,27 +398,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_path_to_error" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -818,9 +409,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -852,41 +443,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "tempfile" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" -dependencies = [ - "cfg-if", - "fastrand", - "libc", - "redox_syscall", - "remove_dir_all", - "winapi", -] - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.25.0" @@ -918,16 +474,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.7" @@ -942,47 +488,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - [[package]] name = "tower-service" version = "0.3.2" @@ -996,7 +501,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", - "log", "pin-project-lite", "tracing-core", ] @@ -1016,44 +520,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" -[[package]] -name = "unicode-bidi" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" - [[package]] name = "unicode-ident" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "url" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "want" version = "0.3.0" @@ -1070,82 +542,6 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "wasm-bindgen" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" - -[[package]] -name = "web-sys" -version = "0.3.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "winapi" version = "0.3.9" @@ -1248,12 +644,3 @@ name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" - -[[package]] -name = "winreg" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" -dependencies = [ - "winapi", -] diff --git a/collector/Cargo.toml b/collector/Cargo.toml index 55edbf4fa58..3875c5e3349 100644 --- a/collector/Cargo.toml +++ b/collector/Cargo.toml @@ -6,8 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axum = "0.6.7" -reqwest = "0.11.14" +hyper = { version = "0.14.24", features = ["full"] } rmp = "0.8.11" rmp-serde = "1.1.1" serde = { version = "1.0.152", features = ["derive"] } diff --git a/collector/src/main.rs b/collector/src/main.rs index 3f8ba6e6031..89fc3912304 100644 --- a/collector/src/main.rs +++ b/collector/src/main.rs @@ -1,17 +1,13 @@ -use axum::body::Bytes; -use axum::extract::{DefaultBodyLimit,State}; -use axum::routing::put; -use axum::Router; +use hyper::{Body, Error, Method, Server, Version}; +use hyper::http::Response; +use hyper::service::{make_service_fn, service_fn}; use rmp::encode; use rmp::encode::ByteBuf; +use serde::Deserialize; use serde_json::Value; -use tokio::sync::mpsc; -use tokio::sync::mpsc::Sender; -use tokio::sync::mpsc::Receiver; use std::collections::HashMap; -use std::net::SocketAddr; -use std::sync::Arc; -use serde::Deserialize; +use tokio::sync::mpsc; +use tokio::sync::mpsc::{Receiver,Sender}; type Traces = HashMap; @@ -39,20 +35,20 @@ struct Trace { } #[derive(Deserialize, Debug)] -struct Payload { - service: String, - events: Value +struct Metadata { + service: String } -struct AppState { - tx: Sender +#[derive(Deserialize, Debug)] +struct Payload { + metadata: Metadata, + events: Vec } -// TODO: Use hyper directly instead of axum/reqwest. // TODO: Decouple processing from transport. -// TODO: Store traces per connection and cleanup on connection close. +// TODO: Cleanup traces on connection close. // TODO: Read MsgPack manually and copy bytes to span buffer directly. -// TODO: Add support for payload metadata (i.e. service, language). +// TODO: Add support for more payload metadata (i.e. language). // TODO: Use string table. // TODO: Read events into structs instead of serde values. // TODO: Event for adding trace tags. @@ -65,53 +61,70 @@ struct AppState { // TODO: Add benchmarks. #[tokio::main] -async fn main() { - let (tx, mut rx): (Sender, Receiver) = mpsc::channel(100); +async fn main() -> Result<(), Box> { + let addr = ([127, 0, 0, 1], 8127).into(); + let make_svc = make_service_fn(|_conn| { + let (tx, mut rx): (Sender, Receiver) = mpsc::channel(100); - let processor = tokio::spawn(async move { - let mut traces = Traces::new(); + tokio::spawn(async move { + let mut traces = Traces::new(); - while let Some(payload) = rx.recv().await { - let events = payload.events.as_array().unwrap().to_vec(); + while let Some(payload) = rx.recv().await { + // println!("{:#?}", payload); - for event in events { - process_event(&mut traces, event, &payload); - } + let metadata = payload.metadata; + let events = payload.events; - flush(&mut traces).await; + for event in events { + process_event(&mut traces, event, &metadata); + } + + flush(&mut traces).await; + } + }); + + async move { + Ok::<_, Error>(service_fn(move |mut req| { + let tx = tx.clone(); + + async move { + let method = req.method(); + let path = req.uri().path(); + let version = req.version(); + + if version == Version::HTTP_11 && method == Method::PUT && path == "/v0.1/events" { + let bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); + let data: Vec = bytes.try_into().unwrap(); + let payload: Payload = rmp_serde::from_slice(&data).unwrap(); + + tx.send(payload).await.unwrap(); + + Ok(Response::new(Body::from(""))) + } else { + Err("Unsupported request") // TODO: not a 500 + } + } + })) } }); - let state = Arc::new(AppState { tx }); - let addr = SocketAddr::from(([127, 0, 0, 1], 8127)); - let app = Router::new() - .route("/v0.1/events", put(handle_events)) - .layer(DefaultBodyLimit::max(1024 * 1024 * 16)) // 16MB - .with_state(state); - - let server = axum::Server::bind(&addr) - .serve(app.into_make_service()); - - let (_, _) = tokio::join!(server, processor); -} + let server = Server::bind(&addr).serve(make_svc); -async fn handle_events(State(state): State>, bytes: Bytes) { - let data: Vec = bytes.try_into().unwrap(); - let payload: Payload = rmp_serde::from_slice(&data).unwrap(); + println!("Listening on http://{}", addr); - // println!("{:#?}", payload); + server.await?; - state.tx.send(payload).await.unwrap(); + Ok(()) } -fn process_event(traces: &mut Traces, event: Value, payload: &Payload) { +fn process_event(traces: &mut Traces, event: Value, metadata: &Metadata) { let event_type = event.get(0).unwrap().as_u64().unwrap(); match event_type { - 1 => process_start_koa_request(traces, &event, payload), + 1 => process_start_koa_request(traces, &event, &metadata), 2 => process_add_error(traces, &event), 3 => process_finish_koa_request(traces, &event), - 4 => process_start_span(traces, &event, payload), + 4 => process_start_span(traces, &event, &metadata), 5 => process_finish_span(traces, &event), 6 => process_add_tags(traces, &event), _ => () @@ -157,7 +170,7 @@ fn process_add_tags(traces: &mut Traces, event: &Value) { } } -fn process_start_span(traces: &mut Traces, event: &Value, payload: &Payload) { +fn process_start_span(traces: &mut Traces, event: &Value, metadata: &Metadata) { let mut meta = HashMap::new(); let mut metrics = HashMap::new(); @@ -182,7 +195,7 @@ fn process_start_span(traces: &mut Traces, event: &Value, payload: &Payload) { parent_id: event[4].as_u64().unwrap(), name: event[6].as_str().unwrap().to_string(), resource: event[7].as_str().unwrap().to_string(), - service: if service.is_empty() { payload.service.to_string() } else { service.to_string() }, + service: if service.is_empty() { metadata.service.to_string() } else { service.to_string() }, error: 0, start: event[1].as_u64().unwrap(), duration: 0, @@ -217,7 +230,7 @@ fn process_finish_span(traces: &mut Traces, event: &Value) { } } -fn process_start_koa_request(traces: &mut Traces, event: &Value, payload: &Payload) { +fn process_start_koa_request(traces: &mut Traces, event: &Value, metadata: &Metadata) { let mut meta = HashMap::new(); let metrics = HashMap::new(); @@ -235,7 +248,7 @@ fn process_start_koa_request(traces: &mut Traces, event: &Value, payload: &Paylo parent_id: event[4].as_u64().unwrap(), name: String::from("koa.request"), resource, - service: payload.service.to_string(), + service: metadata.service.to_string(), error: 0, start: event[1].as_u64().unwrap(), duration: 0, @@ -302,22 +315,21 @@ async fn flush(traces: &mut Traces) { traces.retain(|_, t| t.started != t.finished); - let client = reqwest::Client::new(); + let client = hyper::Client::new(); let data: Vec = wr.as_vec().to_vec(); - - // println!("{:#?}", data); - - client.put("http://localhost:8126/v0.4/traces") + let req = hyper::Request::builder() + .method(hyper::Method::PUT) + .uri("http://localhost:8126/v0.4/traces") .header("Content-Type", "application/msgpack") .header("X-Datadog-Trace-Count", trace_count.to_string()) // .header("Datadog-Meta-Tracer-Version", "") // .header("Datadog-Meta-Lang", "") // .header("Datadog-Meta-Lang-Version", "") // .header("Datadog-Meta-Lang-Interpreter", "") - .body(data) - .send() - .await + .body(hyper::Body::from(data)) .unwrap(); + + client.request(req).await.unwrap(); } } @@ -333,8 +345,6 @@ fn encode_trace(wr: &mut ByteBuf, trace: &Trace) { encode::write_array_len(wr, trace.spans.len() as u32).unwrap(); for span in trace.spans.values() { - // println!("{:#?}", test); - match &span.span_type { Some(span_type) => { encode::write_map_len(wr, 12).unwrap(); From dde23440c40129852d87c551c0bcff90dd8bd580 Mon Sep 17 00:00:00 2001 From: rochdev Date: Wed, 22 Feb 2023 19:07:21 -0500 Subject: [PATCH 04/19] migrate to api compatible with hyper 1.0 --- collector/src/main.rs | 78 +++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/collector/src/main.rs b/collector/src/main.rs index 89fc3912304..dc569f71591 100644 --- a/collector/src/main.rs +++ b/collector/src/main.rs @@ -1,11 +1,14 @@ -use hyper::{Body, Error, Method, Server, Version}; +use hyper::{Body, Method, StatusCode, Error}; use hyper::http::Response; -use hyper::service::{make_service_fn, service_fn}; +use hyper::server::conn::Http; +use hyper::service::service_fn; use rmp::encode; use rmp::encode::ByteBuf; use serde::Deserialize; use serde_json::Value; use std::collections::HashMap; +use std::net::SocketAddr; +use tokio::net::TcpListener; use tokio::sync::mpsc; use tokio::sync::mpsc::{Receiver,Sender}; @@ -46,7 +49,8 @@ struct Payload { } // TODO: Decouple processing from transport. -// TODO: Cleanup traces on connection close. +// TODO: Stream the data somehow. +// TODO: Make sure that traces are cleaned up on connection close. // TODO: Read MsgPack manually and copy bytes to span buffer directly. // TODO: Add support for more payload metadata (i.e. language). // TODO: Use string table. @@ -62,8 +66,11 @@ struct Payload { #[tokio::main] async fn main() -> Result<(), Box> { - let addr = ([127, 0, 0, 1], 8127).into(); - let make_svc = make_service_fn(|_conn| { + let addr = SocketAddr::from(([127, 0, 0, 1], 8127)); + let listener = TcpListener::bind(addr).await?; + + loop { + let (stream, _) = listener.accept().await?; let (tx, mut rx): (Sender, Receiver) = mpsc::channel(100); tokio::spawn(async move { @@ -83,38 +90,37 @@ async fn main() -> Result<(), Box> { } }); - async move { - Ok::<_, Error>(service_fn(move |mut req| { - let tx = tx.clone(); - - async move { - let method = req.method(); - let path = req.uri().path(); - let version = req.version(); - - if version == Version::HTTP_11 && method == Method::PUT && path == "/v0.1/events" { - let bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); - let data: Vec = bytes.try_into().unwrap(); - let payload: Payload = rmp_serde::from_slice(&data).unwrap(); - - tx.send(payload).await.unwrap(); - - Ok(Response::new(Body::from(""))) - } else { - Err("Unsupported request") // TODO: not a 500 + tokio::spawn(async move { + Http::new() + .http1_only(true) + .http1_keep_alive(true) + .serve_connection(stream, service_fn(move |mut req| { + let tx = tx.clone(); + + async move { + let method = req.method(); + let path = req.uri().path(); + let body; + + if method == Method::PUT && path == "/v0.1/events" { + let bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); + let data: Vec = bytes.try_into().unwrap(); + let payload: Payload = rmp_serde::from_slice(&data).unwrap(); + + tx.send(payload).await.unwrap(); + + body = Response::new(Body::from("")); + } else { + body = Response::builder().status(StatusCode::NOT_FOUND).body(Body::from("")).unwrap() + } + + Ok::<_, Error>(body) } - } - })) - } - }); - - let server = Server::bind(&addr).serve(make_svc); - - println!("Listening on http://{}", addr); - - server.await?; - - Ok(()) + })) + .await + .unwrap(); + }); + } } fn process_event(traces: &mut Traces, event: Value, metadata: &Metadata) { From 370a6af7d8ff1249cf594a8ffbe1fd961418057a Mon Sep 17 00:00:00 2001 From: rochdev Date: Sun, 26 Feb 2023 14:45:10 -0500 Subject: [PATCH 05/19] update encoding format and decode manually --- .../plugin-koa/internal-tracer/encoder.js | 62 +-- collector/Cargo.lock | 51 --- collector/Cargo.toml | 10 +- collector/src/main.rs | 359 ++++++++++-------- 4 files changed, 224 insertions(+), 258 deletions(-) diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js index 2f2dbb7cabe..729d36cb399 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -7,7 +7,7 @@ const { Client } = require('./client') const processStartTime = BigInt(Date.now() * 1e6) const processStartTicks = process.hrtime.bigint() const now = () => Number(processStartTime + process.hrtime.bigint() - processStartTicks) -const service = process.env.DD_SERVICE || 'unnamed-node-app' +// const service = process.env.DD_SERVICE || 'unnamed-node-app' const ARRAY_OF_TWO = 0x92 const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB const flushInterval = 2000 @@ -52,8 +52,9 @@ class Encoder { // resource comes from mixing http method and route // error will be its own event - this._encodeFixArray(bytes, 7) + this._encodeFixArray(bytes, 2) this._encodeShort(bytes, eventTypes.KOA_REQUEST_START) // implied: name + this._encodeFixArray(bytes, 6) this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) @@ -70,8 +71,9 @@ class Encoder { if (!store || !store.traceContext) return - this._encodeFixArray(bytes, 5) + this._encodeFixArray(bytes, 2) this._encodeShort(bytes, eventTypes.KOA_REQUEST_FINISH) // implied: name + this._encodeFixArray(bytes, 4) this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) @@ -86,8 +88,9 @@ class Encoder { if (!store || !store.traceContext) return // TODO: support errors without tracing - this._encodeFixArray(bytes, 7) + this._encodeFixArray(bytes, 2) this._encodeShort(bytes, eventTypes.ERROR) // implied: name + this._encodeFixArray(bytes, 6) this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) @@ -98,8 +101,7 @@ class Encoder { this._afterEncode() } - // TODO: support new payload format - makePayload05 () { + makePayload () { const prefixSize = 1 const stringSize = this._stringBytes.length + 5 const eventSize = this._eventBytes.length + 5 @@ -110,31 +112,7 @@ class Encoder { buffer[offset++] = ARRAY_OF_TWO offset = this._writeStrings(buffer, offset) - offset = this._writeEvents(buffer, offset) - - this._reset() - - return buffer - } - - makePayload () { - this._encodeMap(this._metadataBytes, { service }) // HACK - - const metadataBytes = this._metadataBytes - const metadataSize = metadataBytes.length + 9 - const eventSize = this._eventBytes.length + 5 + 7 - const buffer = Buffer.allocUnsafe(1 + metadataSize + eventSize) - - let offset = 0 - - buffer[offset++] = 0x82 // fixmap(2) - - buffer[offset++] = 0xa8 // fixstr(8) - offset += buffer.write('metadata', offset) - offset += metadataBytes.buffer.copy(buffer, offset, 0, metadataBytes.length) - - buffer[offset++] = 0xa6 // fixstr(6) - offset += buffer.write('events', offset) + // TODO: add metadata offset = this._writeEvents(buffer, offset) this._reset() @@ -308,17 +286,9 @@ class Encoder { } } - _encodeString05 (bytes, value = '') { - this._cacheString(value) - this._encodeInteger(bytes, this._stringMap[value]) - } - _encodeString (bytes, value = '') { this._cacheString(value) - - const { start, end } = this._stringMap[value] - - this._stringBytes.copy(bytes, start, end) + this._encodeInteger(bytes, this._stringMap[value]) } _encodeFloat (bytes, value) { @@ -341,23 +311,13 @@ class Encoder { } } - _cacheString05 (value) { + _cacheString (value) { if (!(value in this._stringMap)) { this._stringMap[value] = this._stringCount++ this._stringBytes.write(value) } } - _cacheString (value) { - if (!(value in this._stringMap)) { - this._stringCount++ - this._stringMap[value] = { - start: this._stringBytes.length, - end: this._stringBytes.length + this._stringBytes.write(value) - } - } - } - _writeArrayPrefix (buffer, offset, count) { buffer[offset++] = 0xdd buffer.writeUInt32BE(count, offset) diff --git a/collector/Cargo.lock b/collector/Cargo.lock index b33565841f5..8ea4945ce5d 100644 --- a/collector/Cargo.lock +++ b/collector/Cargo.lock @@ -38,9 +38,6 @@ version = "0.1.0" dependencies = [ "hyper", "rmp", - "rmp-serde", - "serde", - "serde_json", "tokio", ] @@ -344,60 +341,12 @@ dependencies = [ "paste", ] -[[package]] -name = "rmp-serde" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5b13be192e0220b8afb7222aa5813cb62cc269ebb5cac346ca6487681d2913e" -dependencies = [ - "byteorder", - "rmp", - "serde", -] - -[[package]] -name = "ryu" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" - [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "serde" -version = "1.0.152" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.152" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" -dependencies = [ - "itoa", - "ryu", - "serde", -] - [[package]] name = "signal-hook-registry" version = "1.4.1" diff --git a/collector/Cargo.toml b/collector/Cargo.toml index 3875c5e3349..49b8970ec13 100644 --- a/collector/Cargo.toml +++ b/collector/Cargo.toml @@ -3,12 +3,16 @@ name = "dd-trace-collector" version = "0.1.0" edition = "2021" +[profile.release] +codegen-units = 1 +lto = true +opt-level = 3 +panic = "abort" +strip = true + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] hyper = { version = "0.14.24", features = ["full"] } rmp = "0.8.11" -rmp-serde = "1.1.1" -serde = { version = "1.0.152", features = ["derive"] } -serde_json = "1.0.93" tokio = { version = "1.25.0", features = ["full"] } diff --git a/collector/src/main.rs b/collector/src/main.rs index dc569f71591..db80c791af4 100644 --- a/collector/src/main.rs +++ b/collector/src/main.rs @@ -1,12 +1,13 @@ -use hyper::{Body, Method, StatusCode, Error}; +use hyper::body::{Buf, Bytes}; +use hyper::{Body, Method, StatusCode}; use hyper::http::Response; use hyper::server::conn::Http; use hyper::service::service_fn; +use rmp::decode::{NumValueReadError, read_array_len, read_f64, read_map_len, read_str_len}; use rmp::encode; use rmp::encode::ByteBuf; -use serde::Deserialize; -use serde_json::Value; use std::collections::HashMap; +use std::io::Read; use std::net::SocketAddr; use tokio::net::TcpListener; use tokio::sync::mpsc; @@ -37,30 +38,18 @@ struct Trace { spans: HashMap } -#[derive(Deserialize, Debug)] -struct Metadata { - service: String -} - -#[derive(Deserialize, Debug)] -struct Payload { - metadata: Metadata, - events: Vec -} - // TODO: Decouple processing from transport. // TODO: Stream the data somehow. // TODO: Make sure that traces are cleaned up on connection close. -// TODO: Read MsgPack manually and copy bytes to span buffer directly. // TODO: Add support for more payload metadata (i.e. language). -// TODO: Use string table. -// TODO: Read events into structs instead of serde values. +// TODO: Use 0.5 endpoint. // TODO: Event for adding trace tags. // TODO: Event for adding baggage items. // TODO: Add support for sampling. // TODO: Support sending traces directly to Datadog. // TODO: Optimize to minimize allocations and copies. // TODO: Split in modules. +// TODO: Add proper error handling. // TODO: Add tests. // TODO: Add benchmarks. @@ -71,19 +60,27 @@ async fn main() -> Result<(), Box> { loop { let (stream, _) = listener.accept().await?; - let (tx, mut rx): (Sender, Receiver) = mpsc::channel(100); + let (tx, mut rx): (Sender, Receiver) = mpsc::channel(100); tokio::spawn(async move { let mut traces = Traces::new(); while let Some(payload) = rx.recv().await { - // println!("{:#?}", payload); + let mut rd = payload.reader(); + + read_array_len(&mut rd).unwrap(); - let metadata = payload.metadata; - let events = payload.events; + let string_count = read_array_len(&mut rd).unwrap(); + let mut strings: Vec = Vec::with_capacity(string_count as usize); - for event in events { - process_event(&mut traces, event, &metadata); + for _ in 0..string_count { + strings.push(read_str(&mut rd)); + } + + let event_count = read_array_len(&mut rd).unwrap(); + + for _ in 0..event_count { + process_event(&mut traces, &strings, &mut rd); } flush(&mut traces).await; @@ -98,23 +95,26 @@ async fn main() -> Result<(), Box> { let tx = tx.clone(); async move { - let method = req.method(); - let path = req.uri().path(); let body; - if method == Method::PUT && path == "/v0.1/events" { - let bytes = hyper::body::to_bytes(req.body_mut()).await.unwrap(); - let data: Vec = bytes.try_into().unwrap(); - let payload: Payload = rmp_serde::from_slice(&data).unwrap(); - - tx.send(payload).await.unwrap(); - - body = Response::new(Body::from("")); - } else { - body = Response::builder().status(StatusCode::NOT_FOUND).body(Body::from("")).unwrap() + match (req.method(), req.uri().path()) { + (&Method::PUT, "/v0.1/events") => { + // TODO: use body::aggregate instead + let bytes = hyper::body::to_bytes(req.body_mut()).await?; + + tx.send(bytes).await.unwrap(); + + body = Response::new(Body::from("")); + }, + _ => { + body = Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("")) + .unwrap() + } } - Ok::<_, Error>(body) + Ok::<_, hyper::Error>(body) } })) .await @@ -123,140 +123,162 @@ async fn main() -> Result<(), Box> { } } -fn process_event(traces: &mut Traces, event: Value, metadata: &Metadata) { - let event_type = event.get(0).unwrap().as_u64().unwrap(); +fn process_event(traces: &mut Traces, strings: &[String], mut rd: R) { + read_array_len(&mut rd).unwrap(); + + let event_type = read_u64(&mut rd).unwrap(); match event_type { - 1 => process_start_koa_request(traces, &event, &metadata), - 2 => process_add_error(traces, &event), - 3 => process_finish_koa_request(traces, &event), - 4 => process_start_span(traces, &event, &metadata), - 5 => process_finish_span(traces, &event), - 6 => process_add_tags(traces, &event), + 1 => process_start_koa_request(traces, strings, rd), + 2 => process_add_error(traces, strings, rd), + 3 => process_finish_koa_request(traces, strings, rd), + 4 => process_start_span(traces, strings, rd), + 5 => process_finish_span(traces, strings, rd), + 6 => process_add_tags(traces, strings, rd), _ => () } } -fn process_add_error(traces: &mut Traces, event: &Value) { - let maybe_trace = traces.get_mut(&event[2].as_u64().unwrap()); - - match maybe_trace { - Some(trace) => { - let maybe_span = trace.spans.get_mut(&event[3].as_u64().unwrap()); - - match maybe_span { - Some(mut span) => { - span.error = 1; - span.meta.insert(String::from("error.type"), event[4].as_str().unwrap().to_string()); - span.meta.insert(String::from("error.message"), event[5].as_str().unwrap().to_string()); - span.meta.insert(String::from("error.stack"), event[6].as_str().unwrap().to_string()); - }, - None => () - } - }, - None => () - } -} +fn process_add_error(traces: &mut Traces, strings: &[String], mut rd: R) { + let size = read_array_len(&mut rd).unwrap(); + + read_u64(&mut rd).unwrap(); + + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let message = if size >= 4 { + &strings[read_u32(&mut rd).unwrap() as usize] + } else { + "" + }; + let stack = if size >= 5 { + &strings[read_u32(&mut rd).unwrap() as usize] + } else { + "" + }; + let name = if size >= 6 { + &strings[read_u32(&mut rd).unwrap() as usize] + } else { + "" + }; -fn process_add_tags(traces: &mut Traces, event: &Value) { - let maybe_trace = traces.get_mut(&event[2].as_u64().unwrap()); + if let Some(trace) = traces.get_mut(&trace_id) { + if let Some(mut span) = trace.spans.get_mut(&span_id) { + span.error = 1; - match maybe_trace { - Some(trace) => { - let maybe_span = trace.spans.get_mut(&event[3].as_u64().unwrap()); + if !message.is_empty() { + span.meta.insert(String::from("error.message"), String::from(message)); + } + + if !stack.is_empty() { + span.meta.insert(String::from("error.stack"), String::from(stack)); + } - match maybe_span { - Some(span) => { - add_tags_from_value(span, &event[4]); - }, - None => () + if !name.is_empty() { + span.meta.insert(String::from("error.type"), String::from(name)); } - }, - None => () + } } } -fn process_start_span(traces: &mut Traces, event: &Value, metadata: &Metadata) { - let mut meta = HashMap::new(); - let mut metrics = HashMap::new(); +fn process_add_tags(traces: &mut Traces, strings: &[String], mut rd: R) { + read_array_len(&mut rd).unwrap(); + read_u64(&mut rd).unwrap(); - for (k, v) in event[9].as_object().unwrap() { - match v { - Value::Number(v) => { - metrics.insert(k.to_string(), v.as_f64().unwrap()); - }, - Value::String(v) => { - meta.insert(k.to_string(), v.to_string()); - } - _ => () + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let (meta, metrics) = read_tags(&mut rd, strings); + + if let Some(trace) = traces.get_mut(&trace_id) { + if let Some(span) = trace.spans.get_mut(&span_id) { + span.meta.extend(meta); + span.metrics.extend(metrics); } } +} + +fn process_start_span(traces: &mut Traces, strings: &[String], mut rd: R) { + let size = read_array_len(&mut rd).unwrap(); + let start = read_u64(&mut rd).unwrap(); + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let parent_id = read_u64(&mut rd).unwrap(); + let service = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); + let name = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); + let resource = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); + let (meta, metrics) = read_tags(&mut rd, strings); + let span_type: Option = if size >= 10 { + Some(strings[read_u32(&mut rd).unwrap() as usize].to_owned()) + } else { + None + }; - let span_type = event[5].as_str().unwrap(); - let service = event[8].as_str().unwrap(); - let mut span = Span { - span_type: if span_type.is_empty() { None } else { Some(span_type.to_string()) }, - trace_id: event[2].as_u64().unwrap(), - span_id: event[3].as_u64().unwrap(), - parent_id: event[4].as_u64().unwrap(), - name: event[6].as_str().unwrap().to_string(), - resource: event[7].as_str().unwrap().to_string(), - service: if service.is_empty() { metadata.service.to_string() } else { service.to_string() }, + let span = Span { + start, + trace_id, + span_id, + parent_id, + span_type, + name, + resource, + service, error: 0, - start: event[1].as_u64().unwrap(), duration: 0, meta, metrics }; - add_tags_from_value(&mut span, &event[9]); - start_span(traces, span); } -fn process_finish_span(traces: &mut Traces, event: &Value) { - let maybe_trace = traces.get_mut(&event[2].as_u64().unwrap()); +fn process_finish_span(traces: &mut Traces, strings: &[String], mut rd: R) { + read_array_len(&mut rd).unwrap(); - match maybe_trace { - Some(mut trace) => { - let maybe_span = trace.spans.get_mut(&event[3].as_u64().unwrap()); + let start = read_u64(&mut rd).unwrap(); + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let (meta, metrics) = read_tags(&mut rd, strings); - match maybe_span { - Some(mut span) => { - trace.finished += 1; + if let Some(mut trace) = traces.get_mut(&trace_id) { + if let Some(mut span) = trace.spans.get_mut(&span_id) { + trace.finished += 1; - span.duration = event[1].as_u64().unwrap() - span.start; + span.duration = start - span.start; - add_tags_from_value(span, &event[4]); - }, - None => () - } - }, - None => () + span.meta.extend(meta); + span.metrics.extend(metrics); + } } } -fn process_start_koa_request(traces: &mut Traces, event: &Value, metadata: &Metadata) { +fn process_start_koa_request(traces: &mut Traces, strings: &[String], mut rd: R) { let mut meta = HashMap::new(); let metrics = HashMap::new(); - let method = event[5].as_str().unwrap().to_string(); - let url = event[6].as_str().unwrap().to_string(); // TODO: route not url + read_array_len(&mut rd).unwrap(); + + let start = read_u64(&mut rd).unwrap(); + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let parent_id = read_u64(&mut rd).unwrap(); + let method = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); + let url = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); // TODO: route not url + let resource = format!("{method} {url}"); meta.insert(String::from("http.method"), method); meta.insert(String::from("http.url"), url); let span = Span { + start, + trace_id, + span_id, + parent_id, span_type: Some(String::from("web")), - trace_id: event[2].as_u64().unwrap(), - span_id: event[3].as_u64().unwrap(), - parent_id: event[4].as_u64().unwrap(), name: String::from("koa.request"), resource, - service: metadata.service.to_string(), + service: String::from("unnamed-app"), error: 0, - start: event[1].as_u64().unwrap(), duration: 0, meta, metrics @@ -265,25 +287,68 @@ fn process_start_koa_request(traces: &mut Traces, event: &Value, metadata: &Meta start_span(traces, span); } -fn process_finish_koa_request(traces: &mut Traces, event: &Value) { - let maybe_trace = traces.get_mut(&event[2].as_u64().unwrap()); +fn process_finish_koa_request(traces: &mut Traces, _: &[String], mut rd: R) { + read_array_len(&mut rd).unwrap(); - match maybe_trace { - Some(mut trace) => { - let maybe_span = trace.spans.get_mut(&event[3].as_u64().unwrap()); + let start = read_u64(&mut rd).unwrap(); + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let status_code = read_u16(&mut rd).unwrap().to_string(); - match maybe_span { - Some(mut span) => { - trace.finished += 1; + if let Some(mut trace) = traces.get_mut(&trace_id) { + if let Some(mut span) = trace.spans.get_mut(&span_id) { + trace.finished += 1; - span.duration = event[1].as_u64().unwrap() - span.start; - span.meta.insert(String::from("http.status_code"), event[4].as_u64().unwrap().to_string()); - }, - None => () - } - }, - None => () + span.duration = start - span.start; + span.meta.insert(String::from("http.status_code"), status_code); + } + } +} + +fn read_u16(mut rd: R) -> Result { + rmp::decode::read_int(&mut rd) +} + +fn read_u32(mut rd: R) -> Result { + rmp::decode::read_int(&mut rd) +} + +fn read_u64(mut rd: R) -> Result { + rmp::decode::read_int(&mut rd) +} + +fn read_str(mut rd: R) -> String { + let limit = read_str_len(&mut rd).unwrap() as u64; + let mut str = String::new(); + + rd.by_ref().take(limit).read_to_string(&mut str).unwrap(); + + str +} + +fn read_tags(mut rd: R, strings: &[String]) -> (HashMap, HashMap){ + let mut meta = HashMap::new(); + let mut metrics = HashMap::new(); + + let meta_size = read_map_len(&mut rd).unwrap(); + + for _ in 0..meta_size { + meta.insert( + strings[read_u32(&mut rd).unwrap() as usize].to_owned(), + strings[read_u32(&mut rd).unwrap() as usize].to_owned() + ); + } + + let metrics_size = read_map_len(&mut rd).unwrap(); + + for _ in 0..metrics_size { + metrics.insert( + strings[read_u32(&mut rd).unwrap() as usize].to_owned(), + read_f64(&mut rd).unwrap() + ); } + + (meta, metrics) } fn start_span(traces: &mut Traces, span: Span) { @@ -297,26 +362,14 @@ fn start_span(traces: &mut Traces, span: Span) { trace.spans.insert(span.span_id, span); } -fn add_tags_from_value(span: &mut Span, value: &Value) { - for (k, v) in value.as_object().unwrap() { - match v { - Value::Number(v) => { - span.metrics.insert(k.to_string(), v.as_f64().unwrap()); - }, - Value::String(v) => { - span.meta.insert(k.to_string(), v.to_string()); - } - _ => () - } - } -} - async fn flush(traces: &mut Traces) { let mut wr = ByteBuf::new(); let finished_traces: Vec<&Trace> = traces.values().filter(|t| t.started == t.finished).collect(); let trace_count = finished_traces.len(); if trace_count > 0 { + // println!("{:#?}", finished_traces); + encode_traces(&mut wr, finished_traces); traces.retain(|_, t| t.started != t.finished); From 0203c0215f834c772ad618aa8dd5e438ec8668bb Mon Sep 17 00:00:00 2001 From: rochdev Date: Mon, 27 Feb 2023 13:52:45 -0500 Subject: [PATCH 06/19] use the underlying array directly for IDs --- .../sirun/plugin-koa/internal-tracer/encoder.js | 2 -- benchmark/sirun/plugin-koa/internal-tracer/id.js | 15 +++------------ 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js index 729d36cb399..1270c3e1298 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -205,8 +205,6 @@ class Encoder { bytes.reserve(9) bytes.length += 9 - id = id.toArray() - bytes.buffer[offset] = 0xcf bytes.buffer[offset + 1] = id[0] bytes.buffer[offset + 2] = id[1] diff --git a/benchmark/sirun/plugin-koa/internal-tracer/id.js b/benchmark/sirun/plugin-koa/internal-tracer/id.js index 5873f5fa5d5..0822c40f91f 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/id.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/id.js @@ -5,12 +5,7 @@ const { randomFillSync } = require('crypto') const UINT_MAX = 4294967296 const data = new Uint8Array(8 * 8192) -const zeroIdBuffer = new Uint8Array(8) -const zeroId = { - toArray: () => zeroIdBuffer, - toString: () => '0', - toJSON: () => '0' -} +const zeroId = new Uint8Array(8) let batch = 0 @@ -19,11 +14,7 @@ function id (value, raddix) { ? fromNumberString(value, raddix) : pseudoRandom() - return { - toArray: () => buffer, - toString: (raddix) => toNumberString(buffer, raddix), - toJSON: () => toNumberString(buffer) - } + return buffer } function pseudoRandom () { @@ -123,4 +114,4 @@ function writeUInt32BE (buffer, value, offset) { buffer[0 + offset] = value & 255 } -module.exports = { id, zeroId } +module.exports = { id, zeroId, toNumberString } From f28977a03b014f6525dfeda650b5f8d67266f0a1 Mon Sep 17 00:00:00 2001 From: rochdev Date: Mon, 27 Feb 2023 19:57:02 -0500 Subject: [PATCH 07/19] split collector as a multi-module crate --- collector/Cargo.lock | 43 +++- collector/Cargo.toml | 4 +- collector/src/exporting.rs | 9 + collector/src/exporting/agent.rs | 117 +++++++++ collector/src/lib.rs | 4 + collector/src/main.rs | 401 +------------------------------ collector/src/msgpack.rs | 24 ++ collector/src/processing.rs | 271 +++++++++++++++++++++ collector/src/tracing.rs | 26 ++ 9 files changed, 503 insertions(+), 396 deletions(-) create mode 100644 collector/src/exporting.rs create mode 100644 collector/src/exporting/agent.rs create mode 100644 collector/src/lib.rs create mode 100644 collector/src/msgpack.rs create mode 100644 collector/src/processing.rs create mode 100644 collector/src/tracing.rs diff --git a/collector/Cargo.lock b/collector/Cargo.lock index 8ea4945ce5d..ec16bdf42d8 100644 --- a/collector/Cargo.lock +++ b/collector/Cargo.lock @@ -2,6 +2,28 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", +] + +[[package]] +name = "async-trait" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -33,9 +55,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "dd-trace-collector" +name = "ddcollector" version = "0.1.0" dependencies = [ + "async-trait", + "hashbrown 0.13.2", "hyper", "rmp", "tokio", @@ -111,6 +135,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + [[package]] name = "hermit-abi" version = "0.2.6" @@ -185,7 +218,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] @@ -475,6 +508,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "want" version = "0.3.0" diff --git a/collector/Cargo.toml b/collector/Cargo.toml index 49b8970ec13..84111271977 100644 --- a/collector/Cargo.toml +++ b/collector/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "dd-trace-collector" +name = "ddcollector" version = "0.1.0" edition = "2021" @@ -13,6 +13,8 @@ strip = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-trait = "0.1.64" +hashbrown = "0.13.2" hyper = { version = "0.14.24", features = ["full"] } rmp = "0.8.11" tokio = { version = "1.25.0", features = ["full"] } diff --git a/collector/src/exporting.rs b/collector/src/exporting.rs new file mode 100644 index 00000000000..10caac8587c --- /dev/null +++ b/collector/src/exporting.rs @@ -0,0 +1,9 @@ +use crate::tracing::{Traces}; +use async_trait::async_trait; + +pub mod agent; + +#[async_trait] +pub trait Exporter { + async fn export(&self, traces: Traces); +} diff --git a/collector/src/exporting/agent.rs b/collector/src/exporting/agent.rs new file mode 100644 index 00000000000..5ec31208cbb --- /dev/null +++ b/collector/src/exporting/agent.rs @@ -0,0 +1,117 @@ +use crate::tracing::{Trace, Traces}; +use super::Exporter; +use async_trait::async_trait; +use rmp::encode; +use rmp::encode::ByteBuf; +use hashbrown::HashMap; + +pub struct AgentExporter {} + +#[async_trait] +impl Exporter for AgentExporter { + async fn export(&self, traces: Traces) { + let mut wr = ByteBuf::new(); + let trace_count = traces.len(); + + if trace_count > 0 { + // println!("{:#?}", traces); + + self.encode_traces(&mut wr, traces); + + let client = hyper::Client::new(); + let data: Vec = wr.as_vec().to_vec(); + let req = hyper::Request::builder() + .method(hyper::Method::PUT) + .uri("http://localhost:8126/v0.4/traces") + .header("Content-Type", "application/msgpack") + .header("X-Datadog-Trace-Count", trace_count.to_string()) + // .header("Datadog-Meta-Tracer-Version", "") + // .header("Datadog-Meta-Lang", "") + // .header("Datadog-Meta-Lang-Version", "") + // .header("Datadog-Meta-Lang-Interpreter", "") + .body(hyper::Body::from(data)) + .unwrap(); + + client.request(req).await.unwrap(); + } + } +} + +impl Default for AgentExporter { + fn default() -> Self { + Self::new() + } +} + +impl AgentExporter { + pub fn new() -> Self { + Self {} + } + + fn encode_traces(&self, wr: &mut ByteBuf, traces: Traces) { + encode::write_array_len(wr, traces.len() as u32).unwrap(); + + for trace in traces.values() { + self.encode_trace(wr, trace); + } + } + + fn encode_trace(&self, wr: &mut ByteBuf, trace: &Trace) { + encode::write_array_len(wr, trace.spans.len() as u32).unwrap(); + + for span in trace.spans.values() { + match &span.span_type { + Some(span_type) => { + encode::write_map_len(wr, 12).unwrap(); + encode::write_str(wr, "type").unwrap(); + encode::write_str(wr, span_type.as_str()).unwrap(); + }, + None => { + encode::write_map_len(wr, 11).unwrap(); + } + } + + encode::write_str(wr, "trace_id").unwrap(); + encode::write_uint(wr, span.trace_id).unwrap(); + encode::write_str(wr, "span_id").unwrap(); + encode::write_uint(wr, span.span_id).unwrap(); + encode::write_str(wr, "parent_id").unwrap(); + encode::write_uint(wr, span.parent_id).unwrap(); + encode::write_str(wr, "name").unwrap(); + encode::write_str(wr, span.name.as_str()).unwrap(); + encode::write_str(wr, "resource").unwrap(); + encode::write_str(wr, span.resource.as_str()).unwrap(); + encode::write_str(wr, "service").unwrap(); + encode::write_str(wr, span.service.as_str()).unwrap(); + encode::write_str(wr, "error").unwrap(); + encode::write_uint(wr, span.error).unwrap(); + encode::write_str(wr, "start").unwrap(); + encode::write_uint(wr, span.start).unwrap(); + encode::write_str(wr, "duration").unwrap(); + encode::write_uint(wr, span.duration + 1).unwrap(); + + self.encode_meta(wr, &span.meta); + self.encode_metrics(wr, &span.metrics); + } + } + + fn encode_meta(&self, wr: &mut ByteBuf, map: &HashMap) { + encode::write_str(wr, "meta").unwrap(); + encode::write_map_len(wr, map.len() as u32).unwrap(); + + for (k, v) in map { + encode::write_str(wr, k.as_str()).unwrap(); + encode::write_str(wr, v.as_str()).unwrap(); + } + } + + fn encode_metrics(&self, wr: &mut ByteBuf, map: &HashMap) { + encode::write_str(wr, "metrics").unwrap(); + encode::write_map_len(wr, map.len() as u32).unwrap(); + + for (k, v) in map { + encode::write_str(wr, k.as_str()).unwrap(); + encode::write_f64(wr, *v).unwrap(); + } + } +} diff --git a/collector/src/lib.rs b/collector/src/lib.rs new file mode 100644 index 00000000000..aa87006a0c8 --- /dev/null +++ b/collector/src/lib.rs @@ -0,0 +1,4 @@ +pub mod exporting; +pub mod msgpack; +pub mod processing; +pub mod tracing; diff --git a/collector/src/main.rs b/collector/src/main.rs index db80c791af4..02f06bf3e72 100644 --- a/collector/src/main.rs +++ b/collector/src/main.rs @@ -1,54 +1,18 @@ +use ddcollector::exporting::agent::AgentExporter; +use ddcollector::processing::Processor; use hyper::body::{Buf, Bytes}; use hyper::{Body, Method, StatusCode}; use hyper::http::Response; use hyper::server::conn::Http; use hyper::service::service_fn; -use rmp::decode::{NumValueReadError, read_array_len, read_f64, read_map_len, read_str_len}; -use rmp::encode; -use rmp::encode::ByteBuf; -use std::collections::HashMap; -use std::io::Read; use std::net::SocketAddr; use tokio::net::TcpListener; use tokio::sync::mpsc; use tokio::sync::mpsc::{Receiver,Sender}; -type Traces = HashMap; - -#[derive(Debug)] -struct Span { - span_type: Option, - trace_id: u64, - span_id: u64, - parent_id: u64, - name: String, - resource: String, - service: String, - error: u64, - start: u64, - duration: u64, - meta: HashMap, - metrics: HashMap -} - -#[derive(Debug)] -struct Trace { - started: u64, - finished: u64, - spans: HashMap -} - -// TODO: Decouple processing from transport. +// TODO: Move HTTP server to its own module. // TODO: Stream the data somehow. -// TODO: Make sure that traces are cleaned up on connection close. -// TODO: Add support for more payload metadata (i.e. language). -// TODO: Use 0.5 endpoint. -// TODO: Event for adding trace tags. -// TODO: Event for adding baggage items. -// TODO: Add support for sampling. -// TODO: Support sending traces directly to Datadog. -// TODO: Optimize to minimize allocations and copies. -// TODO: Split in modules. +// TODO: Make sure that processor is cleaned up on connection close. // TODO: Add proper error handling. // TODO: Add tests. // TODO: Add benchmarks. @@ -63,27 +27,14 @@ async fn main() -> Result<(), Box> { let (tx, mut rx): (Sender, Receiver) = mpsc::channel(100); tokio::spawn(async move { - let mut traces = Traces::new(); + let exporter = Box::new(AgentExporter::new()); + let mut processor = Processor::new(exporter); while let Some(payload) = rx.recv().await { let mut rd = payload.reader(); - read_array_len(&mut rd).unwrap(); - - let string_count = read_array_len(&mut rd).unwrap(); - let mut strings: Vec = Vec::with_capacity(string_count as usize); - - for _ in 0..string_count { - strings.push(read_str(&mut rd)); - } - - let event_count = read_array_len(&mut rd).unwrap(); - - for _ in 0..event_count { - process_event(&mut traces, &strings, &mut rd); - } - - flush(&mut traces).await; + processor.process(&mut rd); + processor.flush().await; } }); @@ -122,339 +73,3 @@ async fn main() -> Result<(), Box> { }); } } - -fn process_event(traces: &mut Traces, strings: &[String], mut rd: R) { - read_array_len(&mut rd).unwrap(); - - let event_type = read_u64(&mut rd).unwrap(); - - match event_type { - 1 => process_start_koa_request(traces, strings, rd), - 2 => process_add_error(traces, strings, rd), - 3 => process_finish_koa_request(traces, strings, rd), - 4 => process_start_span(traces, strings, rd), - 5 => process_finish_span(traces, strings, rd), - 6 => process_add_tags(traces, strings, rd), - _ => () - } -} - -fn process_add_error(traces: &mut Traces, strings: &[String], mut rd: R) { - let size = read_array_len(&mut rd).unwrap(); - - read_u64(&mut rd).unwrap(); - - let trace_id = read_u64(&mut rd).unwrap(); - let span_id = read_u64(&mut rd).unwrap(); - let message = if size >= 4 { - &strings[read_u32(&mut rd).unwrap() as usize] - } else { - "" - }; - let stack = if size >= 5 { - &strings[read_u32(&mut rd).unwrap() as usize] - } else { - "" - }; - let name = if size >= 6 { - &strings[read_u32(&mut rd).unwrap() as usize] - } else { - "" - }; - - if let Some(trace) = traces.get_mut(&trace_id) { - if let Some(mut span) = trace.spans.get_mut(&span_id) { - span.error = 1; - - if !message.is_empty() { - span.meta.insert(String::from("error.message"), String::from(message)); - } - - if !stack.is_empty() { - span.meta.insert(String::from("error.stack"), String::from(stack)); - } - - if !name.is_empty() { - span.meta.insert(String::from("error.type"), String::from(name)); - } - } - } -} - -fn process_add_tags(traces: &mut Traces, strings: &[String], mut rd: R) { - read_array_len(&mut rd).unwrap(); - read_u64(&mut rd).unwrap(); - - let trace_id = read_u64(&mut rd).unwrap(); - let span_id = read_u64(&mut rd).unwrap(); - let (meta, metrics) = read_tags(&mut rd, strings); - - if let Some(trace) = traces.get_mut(&trace_id) { - if let Some(span) = trace.spans.get_mut(&span_id) { - span.meta.extend(meta); - span.metrics.extend(metrics); - } - } -} - -fn process_start_span(traces: &mut Traces, strings: &[String], mut rd: R) { - let size = read_array_len(&mut rd).unwrap(); - let start = read_u64(&mut rd).unwrap(); - let trace_id = read_u64(&mut rd).unwrap(); - let span_id = read_u64(&mut rd).unwrap(); - let parent_id = read_u64(&mut rd).unwrap(); - let service = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); - let name = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); - let resource = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); - let (meta, metrics) = read_tags(&mut rd, strings); - let span_type: Option = if size >= 10 { - Some(strings[read_u32(&mut rd).unwrap() as usize].to_owned()) - } else { - None - }; - - let span = Span { - start, - trace_id, - span_id, - parent_id, - span_type, - name, - resource, - service, - error: 0, - duration: 0, - meta, - metrics - }; - - start_span(traces, span); -} - -fn process_finish_span(traces: &mut Traces, strings: &[String], mut rd: R) { - read_array_len(&mut rd).unwrap(); - - let start = read_u64(&mut rd).unwrap(); - let trace_id = read_u64(&mut rd).unwrap(); - let span_id = read_u64(&mut rd).unwrap(); - let (meta, metrics) = read_tags(&mut rd, strings); - - if let Some(mut trace) = traces.get_mut(&trace_id) { - if let Some(mut span) = trace.spans.get_mut(&span_id) { - trace.finished += 1; - - span.duration = start - span.start; - - span.meta.extend(meta); - span.metrics.extend(metrics); - } - } -} - -fn process_start_koa_request(traces: &mut Traces, strings: &[String], mut rd: R) { - let mut meta = HashMap::new(); - let metrics = HashMap::new(); - - read_array_len(&mut rd).unwrap(); - - let start = read_u64(&mut rd).unwrap(); - let trace_id = read_u64(&mut rd).unwrap(); - let span_id = read_u64(&mut rd).unwrap(); - let parent_id = read_u64(&mut rd).unwrap(); - let method = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); - let url = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); // TODO: route not url - - let resource = format!("{method} {url}"); - - meta.insert(String::from("http.method"), method); - meta.insert(String::from("http.url"), url); - - let span = Span { - start, - trace_id, - span_id, - parent_id, - span_type: Some(String::from("web")), - name: String::from("koa.request"), - resource, - service: String::from("unnamed-app"), - error: 0, - duration: 0, - meta, - metrics - }; - - start_span(traces, span); -} - -fn process_finish_koa_request(traces: &mut Traces, _: &[String], mut rd: R) { - read_array_len(&mut rd).unwrap(); - - let start = read_u64(&mut rd).unwrap(); - let trace_id = read_u64(&mut rd).unwrap(); - let span_id = read_u64(&mut rd).unwrap(); - let status_code = read_u16(&mut rd).unwrap().to_string(); - - if let Some(mut trace) = traces.get_mut(&trace_id) { - if let Some(mut span) = trace.spans.get_mut(&span_id) { - trace.finished += 1; - - span.duration = start - span.start; - span.meta.insert(String::from("http.status_code"), status_code); - } - } -} - -fn read_u16(mut rd: R) -> Result { - rmp::decode::read_int(&mut rd) -} - -fn read_u32(mut rd: R) -> Result { - rmp::decode::read_int(&mut rd) -} - -fn read_u64(mut rd: R) -> Result { - rmp::decode::read_int(&mut rd) -} - -fn read_str(mut rd: R) -> String { - let limit = read_str_len(&mut rd).unwrap() as u64; - let mut str = String::new(); - - rd.by_ref().take(limit).read_to_string(&mut str).unwrap(); - - str -} - -fn read_tags(mut rd: R, strings: &[String]) -> (HashMap, HashMap){ - let mut meta = HashMap::new(); - let mut metrics = HashMap::new(); - - let meta_size = read_map_len(&mut rd).unwrap(); - - for _ in 0..meta_size { - meta.insert( - strings[read_u32(&mut rd).unwrap() as usize].to_owned(), - strings[read_u32(&mut rd).unwrap() as usize].to_owned() - ); - } - - let metrics_size = read_map_len(&mut rd).unwrap(); - - for _ in 0..metrics_size { - metrics.insert( - strings[read_u32(&mut rd).unwrap() as usize].to_owned(), - read_f64(&mut rd).unwrap() - ); - } - - (meta, metrics) -} - -fn start_span(traces: &mut Traces, span: Span) { - let trace = traces.entry(span.trace_id).or_insert(Trace { - started: 0, - finished: 0, - spans: HashMap::new() - }); - - trace.started += 1; - trace.spans.insert(span.span_id, span); -} - -async fn flush(traces: &mut Traces) { - let mut wr = ByteBuf::new(); - let finished_traces: Vec<&Trace> = traces.values().filter(|t| t.started == t.finished).collect(); - let trace_count = finished_traces.len(); - - if trace_count > 0 { - // println!("{:#?}", finished_traces); - - encode_traces(&mut wr, finished_traces); - - traces.retain(|_, t| t.started != t.finished); - - let client = hyper::Client::new(); - let data: Vec = wr.as_vec().to_vec(); - let req = hyper::Request::builder() - .method(hyper::Method::PUT) - .uri("http://localhost:8126/v0.4/traces") - .header("Content-Type", "application/msgpack") - .header("X-Datadog-Trace-Count", trace_count.to_string()) - // .header("Datadog-Meta-Tracer-Version", "") - // .header("Datadog-Meta-Lang", "") - // .header("Datadog-Meta-Lang-Version", "") - // .header("Datadog-Meta-Lang-Interpreter", "") - .body(hyper::Body::from(data)) - .unwrap(); - - client.request(req).await.unwrap(); - } -} - -fn encode_traces(wr: &mut ByteBuf, traces: Vec<&Trace>) { - encode::write_array_len(wr, traces.len() as u32).unwrap(); - - for trace in traces { - encode_trace(wr, trace); - } -} - -fn encode_trace(wr: &mut ByteBuf, trace: &Trace) { - encode::write_array_len(wr, trace.spans.len() as u32).unwrap(); - - for span in trace.spans.values() { - match &span.span_type { - Some(span_type) => { - encode::write_map_len(wr, 12).unwrap(); - encode::write_str(wr, "type").unwrap(); - encode::write_str(wr, span_type.as_str()).unwrap(); - }, - None => { - encode::write_map_len(wr, 11).unwrap(); - } - } - - encode::write_str(wr, "trace_id").unwrap(); - encode::write_uint(wr, span.trace_id).unwrap(); - encode::write_str(wr, "span_id").unwrap(); - encode::write_uint(wr, span.span_id).unwrap(); - encode::write_str(wr, "parent_id").unwrap(); - encode::write_uint(wr, span.parent_id).unwrap(); - encode::write_str(wr, "name").unwrap(); - encode::write_str(wr, span.name.as_str()).unwrap(); - encode::write_str(wr, "resource").unwrap(); - encode::write_str(wr, span.resource.as_str()).unwrap(); - encode::write_str(wr, "service").unwrap(); - encode::write_str(wr, span.service.as_str()).unwrap(); - encode::write_str(wr, "error").unwrap(); - encode::write_uint(wr, span.error).unwrap(); - encode::write_str(wr, "start").unwrap(); - encode::write_uint(wr, span.start).unwrap(); - encode::write_str(wr, "duration").unwrap(); - encode::write_uint(wr, span.duration + 1).unwrap(); - - encode_meta(wr, &span.meta); - encode_metrics(wr, &span.metrics); - } -} - -fn encode_meta(wr: &mut ByteBuf, map: &HashMap) { - encode::write_str(wr, "meta").unwrap(); - encode::write_map_len(wr, map.len() as u32).unwrap(); - - for (k, v) in map { - encode::write_str(wr, k.as_str()).unwrap(); - encode::write_str(wr, v.as_str()).unwrap(); - } -} - -fn encode_metrics(wr: &mut ByteBuf, map: &HashMap) { - encode::write_str(wr, "metrics").unwrap(); - encode::write_map_len(wr, map.len() as u32).unwrap(); - - for (k, v) in map { - encode::write_str(wr, k.as_str()).unwrap(); - encode::write_f64(wr, *v).unwrap(); - } -} diff --git a/collector/src/msgpack.rs b/collector/src/msgpack.rs new file mode 100644 index 00000000000..3737f028a15 --- /dev/null +++ b/collector/src/msgpack.rs @@ -0,0 +1,24 @@ +use std::io::Read; + +pub use rmp::decode::{NumValueReadError, read_array_len, read_f64, read_map_len, read_str_len}; + +pub fn read_u16(mut rd: R) -> Result { + rmp::decode::read_int(&mut rd) +} + +pub fn read_u32(mut rd: R) -> Result { + rmp::decode::read_int(&mut rd) +} + +pub fn read_u64(mut rd: R) -> Result { + rmp::decode::read_int(&mut rd) +} + +pub fn read_str(mut rd: R) -> String { + let limit = read_str_len(&mut rd).unwrap() as u64; + let mut str = String::new(); + + rd.by_ref().take(limit).read_to_string(&mut str).unwrap(); + + str +} diff --git a/collector/src/processing.rs b/collector/src/processing.rs new file mode 100644 index 00000000000..7d86e595f98 --- /dev/null +++ b/collector/src/processing.rs @@ -0,0 +1,271 @@ +use crate::exporting::Exporter; +use crate::msgpack::{read_array_len, read_f64, read_map_len, read_str, read_u16, read_u32, read_u64}; +use crate::tracing::{Span, Trace, Traces}; +use hashbrown::HashMap; +use std::io::Read; + +pub struct Processor { + exporter: Box, + traces: Traces +} + +// TODO: Decouple processing from exporting. +// TODO: Add support for more payload metadata (i.e. language). +// TODO: Use 0.5 endpoint. +// TODO: Event for adding trace tags. +// TODO: Event for adding baggage items. +// TODO: Add support for sampling. +// TODO: Support sending traces directly to Datadog. +// TODO: Optimize to minimize allocations and copies. + +impl Processor { + pub fn new(exporter: Box) -> Self { + Self { + exporter, + traces: Traces::new() + } + } + + pub fn process(&mut self, mut rd: R) { + read_array_len(&mut rd).unwrap(); + + let string_count = read_array_len(&mut rd).unwrap(); + let mut strings: Vec = Vec::with_capacity(string_count as usize); + + for _ in 0..string_count { + strings.push(read_str(&mut rd)); + } + + let event_count = read_array_len(&mut rd).unwrap(); + + for _ in 0..event_count { + self.process_event(&strings, &mut rd); + } + } + + pub async fn flush(&mut self) { + let finished_traces: HashMap = self.traces + .drain_filter(|_, v| v.started == v.finished) + .collect(); + + self.exporter.export(finished_traces).await; + } + + fn process_event(&mut self, strings: &[String], mut rd: R) { + read_array_len(&mut rd).unwrap(); + + let event_type = read_u64(&mut rd).unwrap(); + + match event_type { + 1 => self.process_start_koa_request(strings, rd), + 2 => self.process_add_error(strings, rd), + 3 => self.process_finish_koa_request(strings, rd), + 4 => self.process_start_span(strings, rd), + 5 => self.process_finish_span(strings, rd), + 6 => self.process_add_tags(strings, rd), + _ => () + } + } + + fn process_add_error(&mut self, strings: &[String], mut rd: R) { + let size = read_array_len(&mut rd).unwrap(); + + read_u64(&mut rd).unwrap(); + + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let message = if size >= 4 { + &strings[read_u32(&mut rd).unwrap() as usize] + } else { + "" + }; + let stack = if size >= 5 { + &strings[read_u32(&mut rd).unwrap() as usize] + } else { + "" + }; + let name = if size >= 6 { + &strings[read_u32(&mut rd).unwrap() as usize] + } else { + "" + }; + + if let Some(trace) = self.traces.get_mut(&trace_id) { + if let Some(mut span) = trace.spans.get_mut(&span_id) { + span.error = 1; + + if !message.is_empty() { + span.meta.insert(String::from("error.message"), String::from(message)); + } + + if !stack.is_empty() { + span.meta.insert(String::from("error.stack"), String::from(stack)); + } + + if !name.is_empty() { + span.meta.insert(String::from("error.type"), String::from(name)); + } + } + } + } + + fn process_add_tags(&mut self, strings: &[String], mut rd: R) { + read_array_len(&mut rd).unwrap(); + read_u64(&mut rd).unwrap(); + + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let (meta, metrics) = self.read_tags(&mut rd, strings); + + if let Some(trace) = self.traces.get_mut(&trace_id) { + if let Some(span) = trace.spans.get_mut(&span_id) { + span.meta.extend(meta); + span.metrics.extend(metrics); + } + } + } + + fn process_start_span(&mut self, strings: &[String], mut rd: R) { + let size = read_array_len(&mut rd).unwrap(); + let start = read_u64(&mut rd).unwrap(); + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let parent_id = read_u64(&mut rd).unwrap(); + let service = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); + let name = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); + let resource = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); + let (meta, metrics) = self.read_tags(&mut rd, strings); + let span_type: Option = if size >= 10 { + Some(strings[read_u32(&mut rd).unwrap() as usize].to_owned()) + } else { + None + }; + + let span = Span { + start, + trace_id, + span_id, + parent_id, + span_type, + name, + resource, + service, + error: 0, + duration: 0, + meta, + metrics + }; + + self.start_span(span); + } + + fn process_finish_span(&mut self, strings: &[String], mut rd: R) { + read_array_len(&mut rd).unwrap(); + + let start = read_u64(&mut rd).unwrap(); + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let (meta, metrics) = self.read_tags(&mut rd, strings); + + if let Some(mut trace) = self.traces.get_mut(&trace_id) { + if let Some(mut span) = trace.spans.get_mut(&span_id) { + trace.finished += 1; + + span.duration = start - span.start; + + span.meta.extend(meta); + span.metrics.extend(metrics); + } + } + } + + fn process_start_koa_request(&mut self, strings: &[String], mut rd: R) { + let mut meta = HashMap::new(); + let metrics = HashMap::new(); + + read_array_len(&mut rd).unwrap(); + + let start = read_u64(&mut rd).unwrap(); + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let parent_id = read_u64(&mut rd).unwrap(); + let method = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); + let url = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); // TODO: route not url + + let resource = format!("{method} {url}"); + + meta.insert(String::from("http.method"), method); + meta.insert(String::from("http.url"), url); + + let span = Span { + start, + trace_id, + span_id, + parent_id, + span_type: Some(String::from("web")), + name: String::from("koa.request"), + resource, + service: String::from("unnamed-app"), + error: 0, + duration: 0, + meta, + metrics + }; + + self.start_span(span); + } + + fn process_finish_koa_request(&mut self, _: &[String], mut rd: R) { + read_array_len(&mut rd).unwrap(); + + let start = read_u64(&mut rd).unwrap(); + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let status_code = read_u16(&mut rd).unwrap().to_string(); + + if let Some(mut trace) = self.traces.get_mut(&trace_id) { + if let Some(mut span) = trace.spans.get_mut(&span_id) { + trace.finished += 1; + + span.duration = start - span.start; + span.meta.insert(String::from("http.status_code"), status_code); + } + } + } + + fn read_tags(&self, mut rd: R, strings: &[String]) -> (HashMap, HashMap){ + let mut meta = HashMap::new(); + let mut metrics = HashMap::new(); + + let meta_size = read_map_len(&mut rd).unwrap(); + + for _ in 0..meta_size { + meta.insert( + strings[read_u32(&mut rd).unwrap() as usize].to_owned(), + strings[read_u32(&mut rd).unwrap() as usize].to_owned() + ); + } + + let metrics_size = read_map_len(&mut rd).unwrap(); + + for _ in 0..metrics_size { + metrics.insert( + strings[read_u32(&mut rd).unwrap() as usize].to_owned(), + read_f64(&mut rd).unwrap() + ); + } + + (meta, metrics) + } + + fn start_span(&mut self, span: Span) { + let trace = self.traces.entry(span.trace_id).or_insert(Trace { + started: 0, + finished: 0, + spans: HashMap::new() + }); + + trace.started += 1; + trace.spans.insert(span.span_id, span); + } +} diff --git a/collector/src/tracing.rs b/collector/src/tracing.rs new file mode 100644 index 00000000000..aecb9b73796 --- /dev/null +++ b/collector/src/tracing.rs @@ -0,0 +1,26 @@ +use hashbrown::HashMap; + +pub type Traces = HashMap; + +#[derive(Debug)] +pub struct Span { + pub span_type: Option, + pub trace_id: u64, + pub span_id: u64, + pub parent_id: u64, + pub name: String, + pub resource: String, + pub service: String, + pub error: u64, + pub start: u64, + pub duration: u64, + pub meta: HashMap, + pub metrics: HashMap +} + +#[derive(Debug)] +pub struct Trace { + pub started: u64, + pub finished: u64, + pub spans: HashMap +} From 8700f5d27d783f7073eacbfdf9e0f0467a30e4ad Mon Sep 17 00:00:00 2001 From: rochdev Date: Mon, 27 Feb 2023 20:07:36 -0500 Subject: [PATCH 08/19] reuse http client between exports --- collector/src/exporting/agent.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/collector/src/exporting/agent.rs b/collector/src/exporting/agent.rs index 5ec31208cbb..c6dd321a047 100644 --- a/collector/src/exporting/agent.rs +++ b/collector/src/exporting/agent.rs @@ -1,11 +1,15 @@ use crate::tracing::{Trace, Traces}; use super::Exporter; use async_trait::async_trait; +use hyper::{Body, Client, Request}; +use hyper::client::HttpConnector; use rmp::encode; use rmp::encode::ByteBuf; use hashbrown::HashMap; -pub struct AgentExporter {} +pub struct AgentExporter { + client: Client +} #[async_trait] impl Exporter for AgentExporter { @@ -18,9 +22,8 @@ impl Exporter for AgentExporter { self.encode_traces(&mut wr, traces); - let client = hyper::Client::new(); let data: Vec = wr.as_vec().to_vec(); - let req = hyper::Request::builder() + let req = Request::builder() .method(hyper::Method::PUT) .uri("http://localhost:8126/v0.4/traces") .header("Content-Type", "application/msgpack") @@ -29,10 +32,10 @@ impl Exporter for AgentExporter { // .header("Datadog-Meta-Lang", "") // .header("Datadog-Meta-Lang-Version", "") // .header("Datadog-Meta-Lang-Interpreter", "") - .body(hyper::Body::from(data)) + .body(Body::from(data)) .unwrap(); - client.request(req).await.unwrap(); + self.client.request(req).await.unwrap(); } } } @@ -45,7 +48,9 @@ impl Default for AgentExporter { impl AgentExporter { pub fn new() -> Self { - Self {} + Self { + client: Client::new() + } } fn encode_traces(&self, wr: &mut ByteBuf, traces: Traces) { From 04218181dfc601f3e455b85171f150b90f5298c1 Mon Sep 17 00:00:00 2001 From: rochdev Date: Tue, 28 Feb 2023 09:53:48 -0500 Subject: [PATCH 09/19] add support for adding strings to the string table from an event --- collector/src/processing.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/collector/src/processing.rs b/collector/src/processing.rs index 7d86e595f98..f83d5c32433 100644 --- a/collector/src/processing.rs +++ b/collector/src/processing.rs @@ -39,7 +39,7 @@ impl Processor { let event_count = read_array_len(&mut rd).unwrap(); for _ in 0..event_count { - self.process_event(&strings, &mut rd); + self.process_event(&mut strings, &mut rd); } } @@ -51,7 +51,7 @@ impl Processor { self.exporter.export(finished_traces).await; } - fn process_event(&mut self, strings: &[String], mut rd: R) { + fn process_event(&mut self, strings: &mut Vec, mut rd: R) { read_array_len(&mut rd).unwrap(); let event_type = read_u64(&mut rd).unwrap(); @@ -63,10 +63,21 @@ impl Processor { 4 => self.process_start_span(strings, rd), 5 => self.process_finish_span(strings, rd), 6 => self.process_add_tags(strings, rd), + 7 => self.process_strings(strings, rd), _ => () } } + fn process_strings(&mut self, strings: &mut Vec, mut rd: R) { + let size = read_array_len(&mut rd).unwrap(); + + strings.reserve(size as usize); + + for _ in 0..size { + strings.push(read_str(&mut rd)); + } + } + fn process_add_error(&mut self, strings: &[String], mut rd: R) { let size = read_array_len(&mut rd).unwrap(); From 1cf607c919b02108ffe7dc9f93301cab5ab91469 Mon Sep 17 00:00:00 2001 From: rochdev Date: Wed, 1 Mar 2023 12:05:14 -0500 Subject: [PATCH 10/19] split in common and server crates --- collector/Cargo.lock | 19 ++++++++++++++----- collector/Cargo.toml | 16 +++++----------- collector/common/Cargo.toml | 13 +++++++++++++ collector/{ => common}/src/exporting.rs | 0 collector/{ => common}/src/exporting/agent.rs | 0 collector/{ => common}/src/lib.rs | 0 collector/{ => common}/src/msgpack.rs | 0 collector/{ => common}/src/processing.rs | 0 collector/{ => common}/src/tracing.rs | 0 collector/server/Cargo.toml | 11 +++++++++++ collector/{ => server}/src/main.rs | 4 ++-- 11 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 collector/common/Cargo.toml rename collector/{ => common}/src/exporting.rs (100%) rename collector/{ => common}/src/exporting/agent.rs (100%) rename collector/{ => common}/src/lib.rs (100%) rename collector/{ => common}/src/msgpack.rs (100%) rename collector/{ => common}/src/processing.rs (100%) rename collector/{ => common}/src/tracing.rs (100%) create mode 100644 collector/server/Cargo.toml rename collector/{ => server}/src/main.rs (96%) diff --git a/collector/Cargo.lock b/collector/Cargo.lock index ec16bdf42d8..2355ee1813b 100644 --- a/collector/Cargo.lock +++ b/collector/Cargo.lock @@ -55,7 +55,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "ddcollector" +name = "common" version = "0.1.0" dependencies = [ "async-trait", @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -380,6 +380,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "server" +version = "0.1.0" +dependencies = [ + "common", + "hyper", + "tokio", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -416,9 +425,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", diff --git a/collector/Cargo.toml b/collector/Cargo.toml index 84111271977..fff574faa8d 100644 --- a/collector/Cargo.toml +++ b/collector/Cargo.toml @@ -1,7 +1,8 @@ -[package] -name = "ddcollector" -version = "0.1.0" -edition = "2021" +[workspace] +members = [ + "common", + "server" +] [profile.release] codegen-units = 1 @@ -11,10 +12,3 @@ panic = "abort" strip = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -async-trait = "0.1.64" -hashbrown = "0.13.2" -hyper = { version = "0.14.24", features = ["full"] } -rmp = "0.8.11" -tokio = { version = "1.25.0", features = ["full"] } diff --git a/collector/common/Cargo.toml b/collector/common/Cargo.toml new file mode 100644 index 00000000000..b1e128aef5a --- /dev/null +++ b/collector/common/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "common" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-trait = "0.1.64" +hashbrown = "0.13.2" +hyper = { version = "0.14.24", features = ["full"] } +rmp = "0.8.11" +tokio = { version = "1.25.0", features = ["full"] } diff --git a/collector/src/exporting.rs b/collector/common/src/exporting.rs similarity index 100% rename from collector/src/exporting.rs rename to collector/common/src/exporting.rs diff --git a/collector/src/exporting/agent.rs b/collector/common/src/exporting/agent.rs similarity index 100% rename from collector/src/exporting/agent.rs rename to collector/common/src/exporting/agent.rs diff --git a/collector/src/lib.rs b/collector/common/src/lib.rs similarity index 100% rename from collector/src/lib.rs rename to collector/common/src/lib.rs diff --git a/collector/src/msgpack.rs b/collector/common/src/msgpack.rs similarity index 100% rename from collector/src/msgpack.rs rename to collector/common/src/msgpack.rs diff --git a/collector/src/processing.rs b/collector/common/src/processing.rs similarity index 100% rename from collector/src/processing.rs rename to collector/common/src/processing.rs diff --git a/collector/src/tracing.rs b/collector/common/src/tracing.rs similarity index 100% rename from collector/src/tracing.rs rename to collector/common/src/tracing.rs diff --git a/collector/server/Cargo.toml b/collector/server/Cargo.toml new file mode 100644 index 00000000000..36570739fa0 --- /dev/null +++ b/collector/server/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +common = { path = "../common" } +hyper = "0.14.24" +tokio = "1.25.0" diff --git a/collector/src/main.rs b/collector/server/src/main.rs similarity index 96% rename from collector/src/main.rs rename to collector/server/src/main.rs index 02f06bf3e72..bf41db9e6cd 100644 --- a/collector/src/main.rs +++ b/collector/server/src/main.rs @@ -1,5 +1,5 @@ -use ddcollector::exporting::agent::AgentExporter; -use ddcollector::processing::Processor; +use common::exporting::agent::AgentExporter; +use common::processing::Processor; use hyper::body::{Buf, Bytes}; use hyper::{Body, Method, StatusCode}; use hyper::http::Response; From b8526f1f0bb8bc0529b5bb8b0543d1379f8d52c4 Mon Sep 17 00:00:00 2001 From: rochdev Date: Wed, 1 Mar 2023 13:11:41 -0500 Subject: [PATCH 11/19] move hyper client to its own crate along with tokio/hyper --- collector/Cargo.lock | 24 +++++++---------- collector/Cargo.toml | 1 + collector/common/Cargo.toml | 3 --- collector/common/src/client.rs | 4 +++ collector/common/src/exporting.rs | 6 ++--- collector/common/src/exporting/agent.rs | 36 +++++++------------------ collector/common/src/lib.rs | 1 + collector/common/src/processing.rs | 4 +-- collector/hyper-client/Cargo.toml | 11 ++++++++ collector/hyper-client/src/lib.rs | 36 +++++++++++++++++++++++++ collector/server/Cargo.toml | 1 + collector/server/src/main.rs | 9 ++++--- 12 files changed, 82 insertions(+), 54 deletions(-) create mode 100644 collector/common/src/client.rs create mode 100644 collector/hyper-client/Cargo.toml create mode 100644 collector/hyper-client/src/lib.rs diff --git a/collector/Cargo.lock b/collector/Cargo.lock index 2355ee1813b..a045ea01c84 100644 --- a/collector/Cargo.lock +++ b/collector/Cargo.lock @@ -13,17 +13,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "async-trait" -version = "0.1.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -58,11 +47,8 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" name = "common" version = "0.1.0" dependencies = [ - "async-trait", "hashbrown 0.13.2", - "hyper", "rmp", - "tokio", ] [[package]] @@ -211,6 +197,15 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-client" +version = "0.1.0" +dependencies = [ + "common", + "hyper", + "tokio", +] + [[package]] name = "indexmap" version = "1.9.2" @@ -386,6 +381,7 @@ version = "0.1.0" dependencies = [ "common", "hyper", + "hyper-client", "tokio", ] diff --git a/collector/Cargo.toml b/collector/Cargo.toml index fff574faa8d..f2f8228c1dd 100644 --- a/collector/Cargo.toml +++ b/collector/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "common", + "hyper-client", "server" ] diff --git a/collector/common/Cargo.toml b/collector/common/Cargo.toml index b1e128aef5a..ebd18d00f58 100644 --- a/collector/common/Cargo.toml +++ b/collector/common/Cargo.toml @@ -6,8 +6,5 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -async-trait = "0.1.64" hashbrown = "0.13.2" -hyper = { version = "0.14.24", features = ["full"] } rmp = "0.8.11" -tokio = { version = "1.25.0", features = ["full"] } diff --git a/collector/common/src/client.rs b/collector/common/src/client.rs new file mode 100644 index 00000000000..d0e7d9cea87 --- /dev/null +++ b/collector/common/src/client.rs @@ -0,0 +1,4 @@ +// TODO: Support streaming with a writer instead of slice. +pub trait Client { + fn request(&self, data: Vec); +} diff --git a/collector/common/src/exporting.rs b/collector/common/src/exporting.rs index 10caac8587c..c8a2c9ee3c7 100644 --- a/collector/common/src/exporting.rs +++ b/collector/common/src/exporting.rs @@ -1,9 +1,7 @@ -use crate::tracing::{Traces}; -use async_trait::async_trait; +use crate::tracing::Traces; pub mod agent; -#[async_trait] pub trait Exporter { - async fn export(&self, traces: Traces); + fn export(&self, traces: Traces); } diff --git a/collector/common/src/exporting/agent.rs b/collector/common/src/exporting/agent.rs index c6dd321a047..72fd49c033e 100644 --- a/collector/common/src/exporting/agent.rs +++ b/collector/common/src/exporting/agent.rs @@ -1,19 +1,16 @@ +use crate::client::Client; use crate::tracing::{Trace, Traces}; use super::Exporter; -use async_trait::async_trait; -use hyper::{Body, Client, Request}; -use hyper::client::HttpConnector; use rmp::encode; use rmp::encode::ByteBuf; use hashbrown::HashMap; pub struct AgentExporter { - client: Client + client: Box } -#[async_trait] impl Exporter for AgentExporter { - async fn export(&self, traces: Traces) { + fn export(&self, traces: Traces) { let mut wr = ByteBuf::new(); let trace_count = traces.len(); @@ -23,33 +20,18 @@ impl Exporter for AgentExporter { self.encode_traces(&mut wr, traces); let data: Vec = wr.as_vec().to_vec(); - let req = Request::builder() - .method(hyper::Method::PUT) - .uri("http://localhost:8126/v0.4/traces") - .header("Content-Type", "application/msgpack") - .header("X-Datadog-Trace-Count", trace_count.to_string()) - // .header("Datadog-Meta-Tracer-Version", "") - // .header("Datadog-Meta-Lang", "") - // .header("Datadog-Meta-Lang-Version", "") - // .header("Datadog-Meta-Lang-Interpreter", "") - .body(Body::from(data)) - .unwrap(); - - self.client.request(req).await.unwrap(); - } - } -} -impl Default for AgentExporter { - fn default() -> Self { - Self::new() + // TODO: Get the response somehow (with a channel?) + // TODO: Make client reusable between requests (with a channel?) + self.client.request(data); + } } } impl AgentExporter { - pub fn new() -> Self { + pub fn new(client: Box) -> Self { Self { - client: Client::new() + client } } diff --git a/collector/common/src/lib.rs b/collector/common/src/lib.rs index aa87006a0c8..72b23c147f8 100644 --- a/collector/common/src/lib.rs +++ b/collector/common/src/lib.rs @@ -1,3 +1,4 @@ +pub mod client; pub mod exporting; pub mod msgpack; pub mod processing; diff --git a/collector/common/src/processing.rs b/collector/common/src/processing.rs index f83d5c32433..93fdaf30cea 100644 --- a/collector/common/src/processing.rs +++ b/collector/common/src/processing.rs @@ -43,12 +43,12 @@ impl Processor { } } - pub async fn flush(&mut self) { + pub fn flush(&mut self) { let finished_traces: HashMap = self.traces .drain_filter(|_, v| v.started == v.finished) .collect(); - self.exporter.export(finished_traces).await; + self.exporter.export(finished_traces); } fn process_event(&mut self, strings: &mut Vec, mut rd: R) { diff --git a/collector/hyper-client/Cargo.toml b/collector/hyper-client/Cargo.toml new file mode 100644 index 00000000000..b1c416ac261 --- /dev/null +++ b/collector/hyper-client/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "hyper-client" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +common = { path = "../common" } +hyper = { version = "0.14.24", features = ["full"] } +tokio = { version = "1.25.0", features = ["full"] } diff --git a/collector/hyper-client/src/lib.rs b/collector/hyper-client/src/lib.rs new file mode 100644 index 00000000000..f87342a295b --- /dev/null +++ b/collector/hyper-client/src/lib.rs @@ -0,0 +1,36 @@ +use common::client::Client; + +pub struct HyperClient {} + +impl HyperClient { + pub fn new() -> Self { + Self {} + } +} + +impl Default for HyperClient { + fn default() -> Self { + Self::new() + } +} + +impl Client for HyperClient { + fn request(&self, data: Vec) { + // TODO: configuration options + let req = hyper::Request::builder() + .method(hyper::Method::PUT) + .uri("http://localhost:8126/v0.5/traces") + .header("Content-Type", "application/msgpack") + // .header("X-Datadog-Trace-Count", trace_count.to_string()) + // .header("Datadog-Meta-Tracer-Version", "") + // .header("Datadog-Meta-Lang", "") + // .header("Datadog-Meta-Lang-Version", "") + // .header("Datadog-Meta-Lang-Interpreter", "") + .body(hyper::Body::from(data)) + .unwrap(); + + tokio::spawn(async move { + hyper::Client::new().request(req).await.unwrap(); + }); + } +} diff --git a/collector/server/Cargo.toml b/collector/server/Cargo.toml index 36570739fa0..752c67997b5 100644 --- a/collector/server/Cargo.toml +++ b/collector/server/Cargo.toml @@ -7,5 +7,6 @@ edition = "2021" [dependencies] common = { path = "../common" } +hyper-client = { path = "../hyper-client" } hyper = "0.14.24" tokio = "1.25.0" diff --git a/collector/server/src/main.rs b/collector/server/src/main.rs index bf41db9e6cd..2b6474aab08 100644 --- a/collector/server/src/main.rs +++ b/collector/server/src/main.rs @@ -5,6 +5,7 @@ use hyper::{Body, Method, StatusCode}; use hyper::http::Response; use hyper::server::conn::Http; use hyper::service::service_fn; +use hyper_client::HyperClient; use std::net::SocketAddr; use tokio::net::TcpListener; use tokio::sync::mpsc; @@ -27,14 +28,14 @@ async fn main() -> Result<(), Box> { let (tx, mut rx): (Sender, Receiver) = mpsc::channel(100); tokio::spawn(async move { - let exporter = Box::new(AgentExporter::new()); - let mut processor = Processor::new(exporter); - while let Some(payload) = rx.recv().await { + let client = Box::new(HyperClient::new()); + let exporter = Box::new(AgentExporter::new(client)); + let mut processor = Processor::new(exporter); let mut rd = payload.reader(); processor.process(&mut rd); - processor.flush().await; + processor.flush(); } }); From 467a4cde3c4068baac246d28bbe2b4f279a45729 Mon Sep 17 00:00:00 2001 From: rochdev Date: Wed, 1 Mar 2023 14:02:19 -0500 Subject: [PATCH 12/19] remove done todo --- collector/server/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/collector/server/src/main.rs b/collector/server/src/main.rs index 2b6474aab08..257c0864c25 100644 --- a/collector/server/src/main.rs +++ b/collector/server/src/main.rs @@ -11,7 +11,6 @@ use tokio::net::TcpListener; use tokio::sync::mpsc; use tokio::sync::mpsc::{Receiver,Sender}; -// TODO: Move HTTP server to its own module. // TODO: Stream the data somehow. // TODO: Make sure that processor is cleaned up on connection close. // TODO: Add proper error handling. From 5e060181542fc0328d8a86a994ddb32af22f799e Mon Sep 17 00:00:00 2001 From: rochdev Date: Wed, 1 Mar 2023 14:35:16 -0500 Subject: [PATCH 13/19] use correct agent endpoint version --- collector/hyper-client/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/hyper-client/src/lib.rs b/collector/hyper-client/src/lib.rs index f87342a295b..f8142f4257a 100644 --- a/collector/hyper-client/src/lib.rs +++ b/collector/hyper-client/src/lib.rs @@ -19,7 +19,7 @@ impl Client for HyperClient { // TODO: configuration options let req = hyper::Request::builder() .method(hyper::Method::PUT) - .uri("http://localhost:8126/v0.5/traces") + .uri("http://localhost:8126/v0.4/traces") .header("Content-Type", "application/msgpack") // .header("X-Datadog-Trace-Count", trace_count.to_string()) // .header("Datadog-Meta-Tracer-Version", "") From 10dc79b882ad92d2d689ded933d26943f78b138e Mon Sep 17 00:00:00 2001 From: rochdev Date: Wed, 1 Mar 2023 14:59:46 -0500 Subject: [PATCH 14/19] cache strings and use 0.5 endpoint --- .../plugin-koa/internal-tracer/encoder.js | 118 +++++++++++++++- benchmark/sirun/plugin-koa/server.js | 6 +- collector/common/src/exporting/agent.rs | 130 +++++++++++------- collector/common/src/msgpack.rs | 4 + collector/common/src/processing.rs | 123 ++++++++++------- collector/common/src/tracing.rs | 25 +++- collector/hyper-client/src/lib.rs | 2 +- 7 files changed, 295 insertions(+), 113 deletions(-) diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js index 1270c3e1298..a4f7ad31bfb 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -15,7 +15,10 @@ const noop = () => {} const eventTypes = { KOA_REQUEST_START: 1, ERROR: 2, - KOA_REQUEST_FINISH: 3 + KOA_REQUEST_FINISH: 3, + START_SPAN: 4, + FINISH_SPAN: 5, + ADD_TAGS: 6 } const float64Array = new Float64Array(1) @@ -41,6 +44,33 @@ class Encoder { return this._eventCount } + // encodeKoaRequestStart (req) { + // const bytes = this._eventBytes + // const store = storage.getStore() + + // if (!store || !store.traceContext) return + + // this._encodeFixArray(bytes, 2) + // this._encodeShort(bytes, eventTypes.START_SPAN) + // this._encodeFixArray(bytes, 10) + // this._encodeLong(bytes, now()) + // this._encodeId(bytes, store.traceContext.traceId) + // this._encodeId(bytes, store.traceContext.spanId) + // this._encodeId(bytes, store.traceContext.parentId) + // this._encodeString(bytes, service) + // this._encodeString(bytes, 'koa.request') + // this._encodeString(bytes, `${req.method} ${req.url}`) + // this._encodeFixMap(bytes, 2) + // this._encodeString(bytes, 'http.method') + // this._encodeString(bytes, req.method) + // this._encodeString(bytes, 'http.url') + // this._encodeString(bytes, req.url) + // this._encodeFixMap(bytes, 0) + // this._encodeString(bytes, 'web') + + // this._afterEncode() + // } + encodeKoaRequestStart (req) { const bytes = this._eventBytes const store = storage.getStore() @@ -53,7 +83,7 @@ class Encoder { // error will be its own event this._encodeFixArray(bytes, 2) - this._encodeShort(bytes, eventTypes.KOA_REQUEST_START) // implied: name + this._encodeUnsigned(bytes, eventTypes.KOA_REQUEST_START) // implied: name this._encodeFixArray(bytes, 6) this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) @@ -65,6 +95,26 @@ class Encoder { this._afterEncode() } + // encodeKoaRequestFinish (res) { + // const bytes = this._eventBytes + // const store = storage.getStore() + + // if (!store || !store.traceContext) return + + // this._encodeFixArray(bytes, 2) + // this._encodeShort(bytes, eventTypes.FINISH_SPAN) + // this._encodeFixArray(bytes, 5) + // this._encodeLong(bytes, now()) + // this._encodeId(bytes, store.traceContext.traceId) + // this._encodeId(bytes, store.traceContext.spanId) + // this._encodeFixMap(bytes, 1) + // this._encodeString(bytes, 'http.status_code') + // this._encodeString(bytes, String(res.statusCode || 0)) + // this._encodeFixMap(bytes, 0) + + // this._afterEncode() + // } + encodeKoaRequestFinish (res) { const bytes = this._eventBytes const store = storage.getStore() @@ -72,12 +122,12 @@ class Encoder { if (!store || !store.traceContext) return this._encodeFixArray(bytes, 2) - this._encodeShort(bytes, eventTypes.KOA_REQUEST_FINISH) // implied: name + this._encodeUnsigned(bytes, eventTypes.KOA_REQUEST_FINISH) // implied: name this._encodeFixArray(bytes, 4) this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) - this._encodeShort(bytes, res.statusCode) + this._encodeUnsigned(bytes, res.statusCode) this._afterEncode() } @@ -181,6 +231,15 @@ class Encoder { bytes.buffer[offset + 4] = length } + _encodeFixMap (bytes, size = 0) { + const offset = bytes.length + + bytes.reserve(1) + bytes.length += 1 + + bytes.buffer[offset] = 0x80 + size + } + _encodeMapPrefix (bytes, keysLength) { const offset = bytes.length @@ -259,6 +318,50 @@ class Encoder { bytes.buffer[offset + 8] = lo } + _encodeUnsigned (bytes, value) { + const offset = bytes.length + + if (value <= 0xff) { + bytes.reserve(2) + bytes.length += 2 + + bytes.buffer[offset] = 0xcc + bytes.buffer[offset + 1] = value + } else if (value <= 0xffff) { + bytes.reserve(3) + bytes.length += 3 + + bytes.buffer[offset] = 0xcd + bytes.buffer[offset + 1] = value >> 8 + bytes.buffer[offset + 2] = value + } else if (value <= 0xffffffff) { + bytes.reserve(5) + bytes.length += 5 + + bytes.buffer[offset] = 0xce + bytes.buffer[offset + 1] = value >> 24 + bytes.buffer[offset + 2] = value >> 16 + bytes.buffer[offset + 3] = value >> 8 + bytes.buffer[offset + 4] = value + } else { + const hi = (value / Math.pow(2, 32)) >> 0 + const lo = value >>> 0 + + bytes.reserve(9) + bytes.length += 9 + + bytes.buffer[offset] = 0xcf + bytes.buffer[offset + 1] = hi >> 24 + bytes.buffer[offset + 2] = hi >> 16 + bytes.buffer[offset + 3] = hi >> 8 + bytes.buffer[offset + 4] = hi + bytes.buffer[offset + 5] = lo >> 24 + bytes.buffer[offset + 6] = lo >> 16 + bytes.buffer[offset + 7] = lo >> 8 + bytes.buffer[offset + 8] = lo + } + } + _encodeMap (bytes, value) { const keys = Object.keys(value) const validKeys = keys.filter(key => typeof value[key] === 'string' || typeof value[key] === 'number') @@ -284,9 +387,14 @@ class Encoder { } } + _encodeFixString (bytes, value = '') { + this._cacheString(value) + this._encodeUnsigned(bytes, this._stringMap[value]) + } + _encodeString (bytes, value = '') { this._cacheString(value) - this._encodeInteger(bytes, this._stringMap[value]) + this._encodeUnsigned(bytes, this._stringMap[value]) } _encodeFloat (bytes, value) { diff --git a/benchmark/sirun/plugin-koa/server.js b/benchmark/sirun/plugin-koa/server.js index 023119a0943..91387345f6e 100644 --- a/benchmark/sirun/plugin-koa/server.js +++ b/benchmark/sirun/plugin-koa/server.js @@ -27,13 +27,17 @@ const requests = parseInt(REQUESTS) let readyServer let total = 0 -app.use(ctx => { +app.use(async ctx => { ctx.body = 'OK' if (++total === requests) { server.close() readyServer.close() } + + await new Promise((resolve) => { + setTimeout(resolve(), 500) + }) }) const server = http.createServer(app.callback()) diff --git a/collector/common/src/exporting/agent.rs b/collector/common/src/exporting/agent.rs index 72fd49c033e..5a12cdcfcd7 100644 --- a/collector/common/src/exporting/agent.rs +++ b/collector/common/src/exporting/agent.rs @@ -1,9 +1,10 @@ use crate::client::Client; -use crate::tracing::{Trace, Traces}; +use crate::tracing::{Trace, Traces, Span, Meta, Metrics}; use super::Exporter; +use hashbrown::HashMap; use rmp::encode; use rmp::encode::ByteBuf; -use hashbrown::HashMap; +use std::rc::Rc; pub struct AgentExporter { client: Box @@ -35,69 +36,104 @@ impl AgentExporter { } } + fn cache_strings(&self, strings: &mut Vec>, positions: &mut HashMap, u32>, trace: &Trace) { + for span in trace.spans.values() { + self.cache_string(strings, positions, &span.service); + self.cache_string(strings, positions, &span.name); + self.cache_string(strings, positions, &span.resource); + self.cache_string(strings, positions, &span.span_type); + + for (k, v) in &span.meta { + self.cache_string(strings, positions, &k); + self.cache_string(strings, positions, &v); + } + + for (k, _) in &span.metrics { + self.cache_string(strings, positions, &k); + } + } + } + + fn cache_string(&self, strings: &mut Vec>, positions: &mut HashMap, u32>, s: &Rc) { + if !positions.contains_key(s) { + let len = strings.len() as u32; + + positions.insert(s.clone(), len); + strings.push(s.clone()); + } + } + + fn encode_strings(&self, wr: &mut ByteBuf, strings: &mut Vec>) { + encode::write_array_len(wr, strings.len() as u32).unwrap(); + + for s in strings { + encode::write_str(wr, s).unwrap(); + } + } + fn encode_traces(&self, wr: &mut ByteBuf, traces: Traces) { + encode::write_array_len(wr, 2).unwrap(); + + let empty_string: Rc = Rc::from(""); + let mut strings = Vec::new(); + let mut positions = HashMap::new(); + + strings.push(empty_string.clone()); + positions.insert(empty_string.clone(), 0u32); + + // TODO: Avoid looping twice over traces/strings. + for trace in traces.values() { + self.cache_strings(&mut strings, &mut positions, trace); + } + + self.encode_strings(wr, &mut strings); + encode::write_array_len(wr, traces.len() as u32).unwrap(); for trace in traces.values() { - self.encode_trace(wr, trace); + self.encode_trace(wr, trace, &positions); } } - fn encode_trace(&self, wr: &mut ByteBuf, trace: &Trace) { + fn encode_trace(&self, wr: &mut ByteBuf, trace: &Trace, positions: &HashMap, u32>) { encode::write_array_len(wr, trace.spans.len() as u32).unwrap(); for span in trace.spans.values() { - match &span.span_type { - Some(span_type) => { - encode::write_map_len(wr, 12).unwrap(); - encode::write_str(wr, "type").unwrap(); - encode::write_str(wr, span_type.as_str()).unwrap(); - }, - None => { - encode::write_map_len(wr, 11).unwrap(); - } - } - - encode::write_str(wr, "trace_id").unwrap(); - encode::write_uint(wr, span.trace_id).unwrap(); - encode::write_str(wr, "span_id").unwrap(); - encode::write_uint(wr, span.span_id).unwrap(); - encode::write_str(wr, "parent_id").unwrap(); - encode::write_uint(wr, span.parent_id).unwrap(); - encode::write_str(wr, "name").unwrap(); - encode::write_str(wr, span.name.as_str()).unwrap(); - encode::write_str(wr, "resource").unwrap(); - encode::write_str(wr, span.resource.as_str()).unwrap(); - encode::write_str(wr, "service").unwrap(); - encode::write_str(wr, span.service.as_str()).unwrap(); - encode::write_str(wr, "error").unwrap(); - encode::write_uint(wr, span.error).unwrap(); - encode::write_str(wr, "start").unwrap(); - encode::write_uint(wr, span.start).unwrap(); - encode::write_str(wr, "duration").unwrap(); - encode::write_uint(wr, span.duration + 1).unwrap(); - - self.encode_meta(wr, &span.meta); - self.encode_metrics(wr, &span.metrics); + self.encode_span(wr, span, positions); } } - fn encode_meta(&self, wr: &mut ByteBuf, map: &HashMap) { - encode::write_str(wr, "meta").unwrap(); - encode::write_map_len(wr, map.len() as u32).unwrap(); + fn encode_span(&self, wr: &mut ByteBuf, span: &Span, positions: &HashMap, u32>) { + encode::write_array_len(wr, 12).unwrap(); + + encode::write_uint(wr, positions[&span.service] as u64).unwrap(); + encode::write_uint(wr, positions[&span.name] as u64).unwrap(); + encode::write_uint(wr, positions[&span.resource] as u64).unwrap(); + encode::write_uint(wr, span.trace_id).unwrap(); + encode::write_uint(wr, span.span_id).unwrap(); + encode::write_uint(wr, span.parent_id).unwrap(); + encode::write_uint(wr, span.start).unwrap(); + encode::write_uint(wr, span.duration + 1).unwrap(); + encode::write_uint(wr, span.error).unwrap(); + self.encode_meta(wr, &span.meta, positions); + self.encode_metrics(wr, &span.metrics, positions); + encode::write_uint(wr, positions[&span.span_type] as u64).unwrap(); + } + + fn encode_meta(&self, wr: &mut ByteBuf, meta: &Meta, positions: &HashMap, u32>) { + encode::write_map_len(wr, meta.len() as u32).unwrap(); - for (k, v) in map { - encode::write_str(wr, k.as_str()).unwrap(); - encode::write_str(wr, v.as_str()).unwrap(); + for (k, v) in meta { + encode::write_uint(wr, positions[k] as u64).unwrap(); + encode::write_uint(wr, positions[v] as u64).unwrap(); } } - fn encode_metrics(&self, wr: &mut ByteBuf, map: &HashMap) { - encode::write_str(wr, "metrics").unwrap(); - encode::write_map_len(wr, map.len() as u32).unwrap(); + fn encode_metrics(&self, wr: &mut ByteBuf, metrics: &Metrics, positions: &HashMap, u32>) { + encode::write_map_len(wr, metrics.len() as u32).unwrap(); - for (k, v) in map { - encode::write_str(wr, k.as_str()).unwrap(); + for (k, v) in metrics { + encode::write_uint(wr, positions[k] as u64).unwrap(); encode::write_f64(wr, *v).unwrap(); } } diff --git a/collector/common/src/msgpack.rs b/collector/common/src/msgpack.rs index 3737f028a15..2bf87a91ae0 100644 --- a/collector/common/src/msgpack.rs +++ b/collector/common/src/msgpack.rs @@ -14,6 +14,10 @@ pub fn read_u64(mut rd: R) -> Result { rmp::decode::read_int(&mut rd) } +pub fn read_usize(mut rd: R) -> Result { + rmp::decode::read_int(&mut rd) +} + pub fn read_str(mut rd: R) -> String { let limit = read_str_len(&mut rd).unwrap() as u64; let mut str = String::new(); diff --git a/collector/common/src/processing.rs b/collector/common/src/processing.rs index 93fdaf30cea..b41c85d4072 100644 --- a/collector/common/src/processing.rs +++ b/collector/common/src/processing.rs @@ -1,12 +1,14 @@ use crate::exporting::Exporter; -use crate::msgpack::{read_array_len, read_f64, read_map_len, read_str, read_u16, read_u32, read_u64}; -use crate::tracing::{Span, Trace, Traces}; -use hashbrown::HashMap; +use crate::msgpack::{read_array_len, read_f64, read_map_len, read_str, read_u16, read_u64, read_usize}; +use crate::tracing::{Span, Trace, Traces, Meta, Metrics}; +use hashbrown::{HashMap, HashSet}; use std::io::Read; +use std::rc::Rc; pub struct Processor { exporter: Box, - traces: Traces + traces: Traces, + strings: HashSet> } // TODO: Decouple processing from exporting. @@ -22,7 +24,20 @@ impl Processor { pub fn new(exporter: Box) -> Self { Self { exporter, - traces: Traces::new() + traces: Traces::new(), + // TODO: Figure out how to cache those properly. + strings: HashSet::from([ + Rc::from(""), + Rc::from("error.message"), + Rc::from("error.stack"), + Rc::from("error.type"), + Rc::from("http.method"), + Rc::from("http.status_code"), + Rc::from("http.url"), + Rc::from("koa.request"), + Rc::from("web"), + Rc::from("unnamed-app") + ]) } } @@ -30,10 +45,10 @@ impl Processor { read_array_len(&mut rd).unwrap(); let string_count = read_array_len(&mut rd).unwrap(); - let mut strings: Vec = Vec::with_capacity(string_count as usize); + let mut strings: Vec> = Vec::with_capacity(string_count as usize); for _ in 0..string_count { - strings.push(read_str(&mut rd)); + strings.push(Rc::from(read_str(&mut rd).as_str())); } let event_count = read_array_len(&mut rd).unwrap(); @@ -51,7 +66,7 @@ impl Processor { self.exporter.export(finished_traces); } - fn process_event(&mut self, strings: &mut Vec, mut rd: R) { + fn process_event(&mut self, strings: &mut Vec>, mut rd: R) { read_array_len(&mut rd).unwrap(); let event_type = read_u64(&mut rd).unwrap(); @@ -68,37 +83,40 @@ impl Processor { } } - fn process_strings(&mut self, strings: &mut Vec, mut rd: R) { + fn process_strings(&mut self, strings: &mut Vec>, mut rd: R) { let size = read_array_len(&mut rd).unwrap(); strings.reserve(size as usize); for _ in 0..size { - strings.push(read_str(&mut rd)); + strings.push(Rc::from(read_str(&mut rd).as_str())); } } - fn process_add_error(&mut self, strings: &[String], mut rd: R) { + // TODO: Update this to use string cache. + // TODO: Store an error object instead of tags on the span. + fn process_add_error(&mut self, strings: &[Rc], mut rd: R) { let size = read_array_len(&mut rd).unwrap(); read_u64(&mut rd).unwrap(); let trace_id = read_u64(&mut rd).unwrap(); let span_id = read_u64(&mut rd).unwrap(); + let message = if size >= 4 { - &strings[read_u32(&mut rd).unwrap() as usize] + strings[read_usize(&mut rd).unwrap()].clone() } else { - "" + Rc::from("") }; let stack = if size >= 5 { - &strings[read_u32(&mut rd).unwrap() as usize] + strings[read_usize(&mut rd).unwrap()].clone() } else { - "" + Rc::from("") }; let name = if size >= 6 { - &strings[read_u32(&mut rd).unwrap() as usize] + strings[read_usize(&mut rd).unwrap()].clone() } else { - "" + Rc::from("") }; if let Some(trace) = self.traces.get_mut(&trace_id) { @@ -106,21 +124,21 @@ impl Processor { span.error = 1; if !message.is_empty() { - span.meta.insert(String::from("error.message"), String::from(message)); + span.meta.insert(Rc::from("error.message"), message); } if !stack.is_empty() { - span.meta.insert(String::from("error.stack"), String::from(stack)); + span.meta.insert(Rc::from("error.stack"), stack); } if !name.is_empty() { - span.meta.insert(String::from("error.type"), String::from(name)); + span.meta.insert(Rc::from("error.type"), name); } } } } - fn process_add_tags(&mut self, strings: &[String], mut rd: R) { + fn process_add_tags(&mut self, strings: &[Rc], mut rd: R) { read_array_len(&mut rd).unwrap(); read_u64(&mut rd).unwrap(); @@ -136,21 +154,18 @@ impl Processor { } } - fn process_start_span(&mut self, strings: &[String], mut rd: R) { - let size = read_array_len(&mut rd).unwrap(); + fn process_start_span(&mut self, strings: &[Rc], mut rd: R) { + read_array_len(&mut rd).unwrap(); + let start = read_u64(&mut rd).unwrap(); let trace_id = read_u64(&mut rd).unwrap(); let span_id = read_u64(&mut rd).unwrap(); let parent_id = read_u64(&mut rd).unwrap(); - let service = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); - let name = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); - let resource = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); + let service = strings[read_usize(&mut rd).unwrap()].clone(); + let name = strings[read_usize(&mut rd).unwrap()].clone(); + let resource = strings[read_usize(&mut rd).unwrap()].clone(); let (meta, metrics) = self.read_tags(&mut rd, strings); - let span_type: Option = if size >= 10 { - Some(strings[read_u32(&mut rd).unwrap() as usize].to_owned()) - } else { - None - }; + let span_type = strings[read_usize(&mut rd).unwrap()].clone(); let span = Span { start, @@ -170,7 +185,7 @@ impl Processor { self.start_span(span); } - fn process_finish_span(&mut self, strings: &[String], mut rd: R) { + fn process_finish_span(&mut self, strings: &[Rc], mut rd: R) { read_array_len(&mut rd).unwrap(); let start = read_u64(&mut rd).unwrap(); @@ -190,7 +205,7 @@ impl Processor { } } - fn process_start_koa_request(&mut self, strings: &[String], mut rd: R) { + fn process_start_koa_request(&mut self, strings: &[Rc], mut rd: R) { let mut meta = HashMap::new(); let metrics = HashMap::new(); @@ -200,23 +215,24 @@ impl Processor { let trace_id = read_u64(&mut rd).unwrap(); let span_id = read_u64(&mut rd).unwrap(); let parent_id = read_u64(&mut rd).unwrap(); - let method = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); - let url = strings[read_u32(&mut rd).unwrap() as usize].to_owned(); // TODO: route not url + let method = strings[read_usize(&mut rd).unwrap()].clone(); + let url = strings[read_usize(&mut rd).unwrap()].clone(); // TODO: route not url - let resource = format!("{method} {url}"); + // TODO: How to cache string concatenation? + let resource = Rc::from(format!("{method} {url}")); - meta.insert(String::from("http.method"), method); - meta.insert(String::from("http.url"), url); + meta.insert(self.from_str("http.method"), method); + meta.insert(self.from_str("http.url"), url); let span = Span { start, trace_id, span_id, parent_id, - span_type: Some(String::from("web")), - name: String::from("koa.request"), + span_type: self.from_str("web"), + name: self.from_str("koa.request"), resource, - service: String::from("unnamed-app"), + service: self.from_str("unnamed-app"), error: 0, duration: 0, meta, @@ -226,25 +242,26 @@ impl Processor { self.start_span(span); } - fn process_finish_koa_request(&mut self, _: &[String], mut rd: R) { + fn process_finish_koa_request(&mut self, _: &[Rc], mut rd: R) { read_array_len(&mut rd).unwrap(); let start = read_u64(&mut rd).unwrap(); let trace_id = read_u64(&mut rd).unwrap(); let span_id = read_u64(&mut rd).unwrap(); - let status_code = read_u16(&mut rd).unwrap().to_string(); + let status_code_key = self.from_str("http.status_code"); + let status_code = Rc::from(read_u16(&mut rd).unwrap().to_string()); if let Some(mut trace) = self.traces.get_mut(&trace_id) { if let Some(mut span) = trace.spans.get_mut(&span_id) { trace.finished += 1; span.duration = start - span.start; - span.meta.insert(String::from("http.status_code"), status_code); + span.meta.insert(status_code_key, status_code); } } } - fn read_tags(&self, mut rd: R, strings: &[String]) -> (HashMap, HashMap){ + fn read_tags(&self, mut rd: R, strings: &[Rc]) -> (Meta, Metrics){ let mut meta = HashMap::new(); let mut metrics = HashMap::new(); @@ -252,8 +269,8 @@ impl Processor { for _ in 0..meta_size { meta.insert( - strings[read_u32(&mut rd).unwrap() as usize].to_owned(), - strings[read_u32(&mut rd).unwrap() as usize].to_owned() + strings[read_usize(&mut rd).unwrap()].clone(), + strings[read_usize(&mut rd).unwrap()].clone() ); } @@ -261,7 +278,7 @@ impl Processor { for _ in 0..metrics_size { metrics.insert( - strings[read_u32(&mut rd).unwrap() as usize].to_owned(), + strings[read_usize(&mut rd).unwrap()].clone(), read_f64(&mut rd).unwrap() ); } @@ -270,13 +287,13 @@ impl Processor { } fn start_span(&mut self, span: Span) { - let trace = self.traces.entry(span.trace_id).or_insert(Trace { - started: 0, - finished: 0, - spans: HashMap::new() - }); + let trace = self.traces.entry(span.trace_id).or_default(); trace.started += 1; trace.spans.insert(span.span_id, span); } + + fn from_str(&self, s: &str) -> Rc { + self.strings.get(s).unwrap().clone() + } } diff --git a/collector/common/src/tracing.rs b/collector/common/src/tracing.rs index aecb9b73796..726eeebe715 100644 --- a/collector/common/src/tracing.rs +++ b/collector/common/src/tracing.rs @@ -1,21 +1,24 @@ use hashbrown::HashMap; +use std::rc::Rc; +pub type Meta = HashMap, Rc>; +pub type Metrics = HashMap, f64>; pub type Traces = HashMap; #[derive(Debug)] pub struct Span { - pub span_type: Option, + pub span_type: Rc, pub trace_id: u64, pub span_id: u64, pub parent_id: u64, - pub name: String, - pub resource: String, - pub service: String, + pub name: Rc, + pub resource: Rc, + pub service: Rc, pub error: u64, pub start: u64, pub duration: u64, - pub meta: HashMap, - pub metrics: HashMap + pub meta: Meta, + pub metrics: Metrics } #[derive(Debug)] @@ -24,3 +27,13 @@ pub struct Trace { pub finished: u64, pub spans: HashMap } + +impl Default for Trace { + fn default() -> Self { + Self { + started: 0, + finished: 0, + spans: HashMap::new() + } + } +} diff --git a/collector/hyper-client/src/lib.rs b/collector/hyper-client/src/lib.rs index f8142f4257a..f87342a295b 100644 --- a/collector/hyper-client/src/lib.rs +++ b/collector/hyper-client/src/lib.rs @@ -19,7 +19,7 @@ impl Client for HyperClient { // TODO: configuration options let req = hyper::Request::builder() .method(hyper::Method::PUT) - .uri("http://localhost:8126/v0.4/traces") + .uri("http://localhost:8126/v0.5/traces") .header("Content-Type", "application/msgpack") // .header("X-Datadog-Trace-Count", trace_count.to_string()) // .header("Datadog-Meta-Tracer-Version", "") From 0e7ac4b27017933be1c8f55143a9873ac1b82726 Mon Sep 17 00:00:00 2001 From: rochdev Date: Thu, 2 Mar 2023 13:40:32 -0500 Subject: [PATCH 15/19] add fake db to benchmark and update event support --- .../plugin-koa/internal-tracer/context.js | 19 +++ .../plugin-koa/internal-tracer/encoder.js | 161 +++++++++--------- .../sirun/plugin-koa/internal-tracer/trace.js | 24 +-- benchmark/sirun/plugin-koa/server.js | 77 ++++++++- collector/common/src/processing.rs | 137 +++++++++------ 5 files changed, 263 insertions(+), 155 deletions(-) create mode 100644 benchmark/sirun/plugin-koa/internal-tracer/context.js diff --git a/benchmark/sirun/plugin-koa/internal-tracer/context.js b/benchmark/sirun/plugin-koa/internal-tracer/context.js new file mode 100644 index 00000000000..46d193b864a --- /dev/null +++ b/benchmark/sirun/plugin-koa/internal-tracer/context.js @@ -0,0 +1,19 @@ +'use strict' + +const { id, zeroId } = require('./id') + +class TraceContext { + constructor (childOf) { + if (childOf) { + this.traceId = childOf.traceId + this.spanId = id() + this.parentId = childOf.spanId + } else { + this.traceId = id() + this.spanId = this.traceId + this.parentId = zeroId + } + } +} + +module.exports = { TraceContext } diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js index a4f7ad31bfb..1bef90fc9b0 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -3,6 +3,7 @@ const Chunk = require('../../../../packages/dd-trace/src/encode/chunk') const { storage } = require('../../../../packages/datadog-core') const { Client } = require('./client') +const { zeroId } = require('./id') const processStartTime = BigInt(Date.now() * 1e6) const processStartTicks = process.hrtime.bigint() @@ -13,12 +14,13 @@ const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB const flushInterval = 2000 const noop = () => {} const eventTypes = { - KOA_REQUEST_START: 1, + WEB_REQUEST_START: 1, ERROR: 2, - KOA_REQUEST_FINISH: 3, + WEB_REQUEST_FINISH: 3, START_SPAN: 4, FINISH_SPAN: 5, - ADD_TAGS: 6 + ADD_TAGS: 6, + MYSQL_START_SPAN: 8 } const float64Array = new Float64Array(1) @@ -44,90 +46,80 @@ class Encoder { return this._eventCount } - // encodeKoaRequestStart (req) { - // const bytes = this._eventBytes - // const store = storage.getStore() - - // if (!store || !store.traceContext) return - - // this._encodeFixArray(bytes, 2) - // this._encodeShort(bytes, eventTypes.START_SPAN) - // this._encodeFixArray(bytes, 10) - // this._encodeLong(bytes, now()) - // this._encodeId(bytes, store.traceContext.traceId) - // this._encodeId(bytes, store.traceContext.spanId) - // this._encodeId(bytes, store.traceContext.parentId) - // this._encodeString(bytes, service) - // this._encodeString(bytes, 'koa.request') - // this._encodeString(bytes, `${req.method} ${req.url}`) - // this._encodeFixMap(bytes, 2) - // this._encodeString(bytes, 'http.method') - // this._encodeString(bytes, req.method) - // this._encodeString(bytes, 'http.url') - // this._encodeString(bytes, req.url) - // this._encodeFixMap(bytes, 0) - // this._encodeString(bytes, 'web') - - // this._afterEncode() - // } - - encodeKoaRequestStart (req) { + encodeWebRequestStart (req, component) { const bytes = this._eventBytes const store = storage.getStore() if (!store || !store.traceContext) return - // service name comes from process - // span name comes from event type - // resource comes from mixing http method and route - // error will be its own event - this._encodeFixArray(bytes, 2) - this._encodeUnsigned(bytes, eventTypes.KOA_REQUEST_START) // implied: name - this._encodeFixArray(bytes, 6) + this._encodeByte(bytes, eventTypes.WEB_REQUEST_START) + this._encodeFixArray(bytes, 8) this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) - this._encodeId(bytes, store.traceContext.parentId) // ??? + this._encodeId(bytes, store.traceContext.parentId) + this._encodeString(bytes, component) this._encodeString(bytes, req.method) this._encodeString(bytes, req.url) + this._encodeString(bytes, req.url) // route this._afterEncode() } - // encodeKoaRequestFinish (res) { - // const bytes = this._eventBytes - // const store = storage.getStore() + encodeWebRequestFinish (res) { + const bytes = this._eventBytes + const store = storage.getStore() - // if (!store || !store.traceContext) return + if (!store || !store.traceContext) return - // this._encodeFixArray(bytes, 2) - // this._encodeShort(bytes, eventTypes.FINISH_SPAN) - // this._encodeFixArray(bytes, 5) - // this._encodeLong(bytes, now()) - // this._encodeId(bytes, store.traceContext.traceId) - // this._encodeId(bytes, store.traceContext.spanId) - // this._encodeFixMap(bytes, 1) - // this._encodeString(bytes, 'http.status_code') - // this._encodeString(bytes, String(res.statusCode || 0)) - // this._encodeFixMap(bytes, 0) + this._encodeFixArray(bytes, 2) + this._encodeByte(bytes, eventTypes.WEB_REQUEST_FINISH) + this._encodeFixArray(bytes, 3) + this._encodeLong(bytes, now()) + this._encodeId(bytes, store.traceContext.traceId) + this._encodeId(bytes, store.traceContext.spanId) + this._encodeShort(bytes, res.statusCode) - // this._afterEncode() - // } + this._afterEncode() + } - encodeKoaRequestFinish (res) { + encodeMysqlQueryStart (query) { const bytes = this._eventBytes const store = storage.getStore() if (!store || !store.traceContext) return this._encodeFixArray(bytes, 2) - this._encodeUnsigned(bytes, eventTypes.KOA_REQUEST_FINISH) // implied: name - this._encodeFixArray(bytes, 4) + this._encodeByte(bytes, eventTypes.MYSQL_START_SPAN) + this._encodeFixArray(bytes, 9) this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) - this._encodeUnsigned(bytes, res.statusCode) + this._encodeId(bytes, store.traceContext.parentId) + this._encodeString(bytes, query.sql) + this._encodeString(bytes, query.conf.database) + this._encodeString(bytes, query.conf.user) + this._encodeString(bytes, query.conf.host) + this._encodeString(bytes, query.conf.port) + + this._afterEncode() + } + + encodeFinish () { + const bytes = this._eventBytes + const store = storage.getStore() + + if (!store || !store.traceContext) return + + this._encodeFixArray(bytes, 2) + this._encodeByte(bytes, eventTypes.FINISH_SPAN) + this._encodeFixArray(bytes, 5) + this._encodeLong(bytes, now()) + this._encodeId(bytes, store.traceContext.traceId) + this._encodeId(bytes, store.traceContext.spanId) + this._encodeFixMap(bytes, 0) + this._encodeFixMap(bytes, 0) this._afterEncode() } @@ -139,14 +131,17 @@ class Encoder { if (!store || !store.traceContext) return // TODO: support errors without tracing this._encodeFixArray(bytes, 2) - this._encodeShort(bytes, eventTypes.ERROR) // implied: name - this._encodeFixArray(bytes, 6) + this._encodeByte(bytes, eventTypes.ERROR) // implied: name + this._encodeFixArray(bytes, error ? 6 : 3) this._encodeLong(bytes, now()) this._encodeId(bytes, store.traceContext.traceId) this._encodeId(bytes, store.traceContext.spanId) - this._encodeString(bytes, error.name) - this._encodeString(bytes, error.message) - this._encodeString(bytes, error.stack) + + if (error) { + this._encodeString(bytes, error.name) + this._encodeString(bytes, error.message) + this._encodeString(bytes, error.stack) + } this._afterEncode() } @@ -261,18 +256,25 @@ class Encoder { _encodeId (bytes, id) { const offset = bytes.length - bytes.reserve(9) - bytes.length += 9 + if (id === zeroId) { + bytes.reserve(1) + bytes.length += 1 - bytes.buffer[offset] = 0xcf - bytes.buffer[offset + 1] = id[0] - bytes.buffer[offset + 2] = id[1] - bytes.buffer[offset + 3] = id[2] - bytes.buffer[offset + 4] = id[3] - bytes.buffer[offset + 5] = id[4] - bytes.buffer[offset + 6] = id[5] - bytes.buffer[offset + 7] = id[6] - bytes.buffer[offset + 8] = id[7] + bytes.buffer[offset] = 0x00 + } else { + bytes.reserve(9) + bytes.length += 9 + + bytes.buffer[offset] = 0xcf + bytes.buffer[offset + 1] = id[0] + bytes.buffer[offset + 2] = id[1] + bytes.buffer[offset + 3] = id[2] + bytes.buffer[offset + 4] = id[3] + bytes.buffer[offset + 5] = id[4] + bytes.buffer[offset + 6] = id[5] + bytes.buffer[offset + 7] = id[6] + bytes.buffer[offset + 8] = id[7] + } } _encodeInteger (bytes, value) { @@ -321,7 +323,12 @@ class Encoder { _encodeUnsigned (bytes, value) { const offset = bytes.length - if (value <= 0xff) { + if (value <= 0x7f) { + bytes.reserve(1) + bytes.length += 1 + + bytes.buffer[offset] = value + } else if (value <= 0xff) { bytes.reserve(2) bytes.length += 2 @@ -446,4 +453,4 @@ class Encoder { } } -module.exports = { Encoder } +module.exports = { Encoder, encoder: new Encoder() } diff --git a/benchmark/sirun/plugin-koa/internal-tracer/trace.js b/benchmark/sirun/plugin-koa/internal-tracer/trace.js index 85d7950225f..932fec1ba31 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/trace.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/trace.js @@ -1,37 +1,21 @@ 'use strict' const { channel } = require('diagnostics_channel') -const { Encoder } = require('./encoder') +const { encoder } = require('./encoder') +const { TraceContext } = require('./context') const { storage } = require('../../../../packages/datadog-core') -const { id, zeroId } = require('./id') const startChannel = channel('apm:koa:request:start') const errorChannel = channel('apm:koa:request:error') const asyncEndChannel = channel('apm:koa:request:async-end') -const encoder = new Encoder() - -class TraceContext { - constructor (childOf) { - if (childOf) { - this.traceId = childOf.traceId - this.spanId = id() - this.parentId = childOf.spanId - } else { - this.traceId = id() - this.spanId = this.traceId - this.parentId = zeroId - } - } -} - startChannel.subscribe(({ req }) => { const store = storage.getStore() const traceContext = new TraceContext(store.traceContext) store.traceContext = traceContext - encoder.encodeKoaRequestStart(req) + encoder.encodeWebRequestStart(req, 'koa') }) errorChannel.subscribe(error => { @@ -39,7 +23,7 @@ errorChannel.subscribe(error => { }) asyncEndChannel.subscribe(({ res }) => { - encoder.encodeKoaRequestFinish(res) + encoder.encodeWebRequestFinish(res, 'koa') // TODO: restore parent context }) diff --git a/benchmark/sirun/plugin-koa/server.js b/benchmark/sirun/plugin-koa/server.js index 91387345f6e..c784753d1e9 100644 --- a/benchmark/sirun/plugin-koa/server.js +++ b/benchmark/sirun/plugin-koa/server.js @@ -1,9 +1,14 @@ 'use strict' -const { PORT, REQUESTS, WITH_INTERNAL_TRACER, WITH_TRACER } = process.env +const { PORT, REQUESTS, WITH_FAKE_DB, WITH_INTERNAL_TRACER, WITH_TRACER } = process.env + +let tracer +let encoder +let storage +let TraceContext if (WITH_TRACER === 'true') { - const tracer = require('../../..') + tracer = require('../../..') tracer.init({ startupLogs: false, plugins: false @@ -14,6 +19,9 @@ if (WITH_TRACER === 'true') { if (WITH_INTERNAL_TRACER === 'true') { require('./internal-tracer') + encoder = require('./internal-tracer/encoder').encoder + storage = require('../../../packages/datadog-core').storage + TraceContext = require('./internal-tracer/context').TraceContext } const http = require('http') @@ -27,7 +35,7 @@ const requests = parseInt(REQUESTS) let readyServer let total = 0 -app.use(async ctx => { +app.use(ctx => { ctx.body = 'OK' if (++total === requests) { @@ -35,9 +43,27 @@ app.use(async ctx => { readyServer.close() } - await new Promise((resolve) => { - setTimeout(resolve(), 500) - }) + if (WITH_FAKE_DB === 'true') { + for (let i = 0; i < 25; i++) { + const query = startQuery() + + if (WITH_TRACER) { + const span = traceQuery(query) + runQuery(query) + span.finish() + } else if (WITH_INTERNAL_TRACER) { + const store = storage.getStore() + const parent = store.traceContext + store.traceContext = new TraceContext(parent) + encoder.encodeMysqlQueryStart(query) + runQuery(query) + encoder.encodeFinish() + store.traceContext = parent + } else { + runQuery(query) + } + } + } }) const server = http.createServer(app.callback()) @@ -46,3 +72,42 @@ server.listen(port, () => { readyServer = net.createServer(() => {}) readyServer.listen(port + 1) }) + +function startQuery () { + return { + sql: 'SELECT * FROM mytable WHERE 1 = 1;', + conf: { + user: 'myuser', + database: 'mydatabase', + host: '127.0.0.1', + port: '3306' + } + } +} + +function runQuery () { + return new Promise(resolve => { + setTimeout(resolve, Math.random() * 5) + }) +} + +function traceQuery (query) { + const childOf = tracer.scope().active() + const span = tracer.startSpan('mysql.query', { + childOf, + tags: { + 'span.kind': 'client', + 'span.type': 'sql', + 'resource.name': query.sql, + 'db.type': 'mysql', + 'db.user': query.conf.user, + 'db.name': query.conf.database, + 'out.host': query.conf.host, + 'out.port': query.conf.port + } + }) + + span.finish() + + return span +} diff --git a/collector/common/src/processing.rs b/collector/common/src/processing.rs index b41c85d4072..173ff0cb33c 100644 --- a/collector/common/src/processing.rs +++ b/collector/common/src/processing.rs @@ -13,7 +13,9 @@ pub struct Processor { // TODO: Decouple processing from exporting. // TODO: Add support for more payload metadata (i.e. language). -// TODO: Use 0.5 endpoint. +// TODO: Custom more efficient events depending on span type. +// TODO: Store service metadata that can be used on every span like service name. +// TODO: Cache things like outgoing host/port or MySQL connection information. // TODO: Event for adding trace tags. // TODO: Event for adding baggage items. // TODO: Add support for sampling. @@ -26,18 +28,7 @@ impl Processor { exporter, traces: Traces::new(), // TODO: Figure out how to cache those properly. - strings: HashSet::from([ - Rc::from(""), - Rc::from("error.message"), - Rc::from("error.stack"), - Rc::from("error.type"), - Rc::from("http.method"), - Rc::from("http.status_code"), - Rc::from("http.url"), - Rc::from("koa.request"), - Rc::from("web"), - Rc::from("unnamed-app") - ]) + strings: HashSet::from([Rc::from("")]) } } @@ -72,13 +63,14 @@ impl Processor { let event_type = read_u64(&mut rd).unwrap(); match event_type { - 1 => self.process_start_koa_request(strings, rd), + 1 => self.process_start_web_request(strings, rd), 2 => self.process_add_error(strings, rd), - 3 => self.process_finish_koa_request(strings, rd), + 3 => self.process_finish_web_request(strings, rd), 4 => self.process_start_span(strings, rd), 5 => self.process_finish_span(strings, rd), 6 => self.process_add_tags(strings, rd), 7 => self.process_strings(strings, rd), + 8 => self.process_start_mysql_query(strings, rd), _ => () } } @@ -93,7 +85,6 @@ impl Processor { } } - // TODO: Update this to use string cache. // TODO: Store an error object instead of tags on the span. fn process_add_error(&mut self, strings: &[Rc], mut rd: R) { let size = read_array_len(&mut rd).unwrap(); @@ -103,36 +94,27 @@ impl Processor { let trace_id = read_u64(&mut rd).unwrap(); let span_id = read_u64(&mut rd).unwrap(); - let message = if size >= 4 { - strings[read_usize(&mut rd).unwrap()].clone() - } else { - Rc::from("") - }; - let stack = if size >= 5 { - strings[read_usize(&mut rd).unwrap()].clone() - } else { - Rc::from("") - }; - let name = if size >= 6 { - strings[read_usize(&mut rd).unwrap()].clone() - } else { - Rc::from("") - }; - - if let Some(trace) = self.traces.get_mut(&trace_id) { - if let Some(mut span) = trace.spans.get_mut(&span_id) { - span.error = 1; - - if !message.is_empty() { - span.meta.insert(Rc::from("error.message"), message); - } - - if !stack.is_empty() { - span.meta.insert(Rc::from("error.stack"), stack); + if size < 4 { + if let Some(trace) = self.traces.get_mut(&trace_id) { + if let Some(mut span) = trace.spans.get_mut(&span_id) { + span.error = 1; } - - if !name.is_empty() { - span.meta.insert(Rc::from("error.type"), name); + } + } else { + let name_key = self.from_str("error.name"); + let name = strings[read_usize(&mut rd).unwrap()].clone(); + let message_key = self.from_str("error.message"); + let message = strings[read_usize(&mut rd).unwrap()].clone(); + let stack_key = self.from_str("error.stack"); + let stack = strings[read_usize(&mut rd).unwrap()].clone(); + + if let Some(trace) = self.traces.get_mut(&trace_id) { + if let Some(mut span) = trace.spans.get_mut(&span_id) { + span.error = 1; + + span.meta.insert(name_key, name); + span.meta.insert(message_key, message); + span.meta.insert(stack_key, stack); } } } @@ -205,7 +187,7 @@ impl Processor { } } - fn process_start_koa_request(&mut self, strings: &[Rc], mut rd: R) { + fn process_start_web_request(&mut self, strings: &[Rc], mut rd: R) { let mut meta = HashMap::new(); let metrics = HashMap::new(); @@ -215,11 +197,14 @@ impl Processor { let trace_id = read_u64(&mut rd).unwrap(); let span_id = read_u64(&mut rd).unwrap(); let parent_id = read_u64(&mut rd).unwrap(); + let component = strings[read_usize(&mut rd).unwrap()].clone(); let method = strings[read_usize(&mut rd).unwrap()].clone(); - let url = strings[read_usize(&mut rd).unwrap()].clone(); // TODO: route not url + let url = strings[read_usize(&mut rd).unwrap()].clone(); + let route = strings[read_usize(&mut rd).unwrap()].clone(); // TODO: How to cache string concatenation? - let resource = Rc::from(format!("{method} {url}")); + let name = Rc::from(format!("{component}.request")); + let resource = Rc::from(format!("{method} {route}")); meta.insert(self.from_str("http.method"), method); meta.insert(self.from_str("http.url"), url); @@ -230,7 +215,7 @@ impl Processor { span_id, parent_id, span_type: self.from_str("web"), - name: self.from_str("koa.request"), + name, resource, service: self.from_str("unnamed-app"), error: 0, @@ -242,7 +227,48 @@ impl Processor { self.start_span(span); } - fn process_finish_koa_request(&mut self, _: &[Rc], mut rd: R) { + fn process_start_mysql_query(&mut self, strings: &[Rc], mut rd: R) { + let mut meta = HashMap::new(); + let mut metrics = HashMap::new(); + + read_array_len(&mut rd).unwrap(); + + let start = read_u64(&mut rd).unwrap(); + let trace_id = read_u64(&mut rd).unwrap(); + let span_id = read_u64(&mut rd).unwrap(); + let parent_id = read_u64(&mut rd).unwrap(); + let sql = strings[read_usize(&mut rd).unwrap()].clone(); + let database = strings[read_usize(&mut rd).unwrap()].clone(); + let user = strings[read_usize(&mut rd).unwrap()].clone(); + let host = strings[read_usize(&mut rd).unwrap()].clone(); + let port = read_u16(&mut rd).unwrap(); + + // TODO: How to cache string concatenation? + meta.insert(self.from_str("db.type"), self.from_str("mysql")); + meta.insert(self.from_str("db.user"), user); + meta.insert(self.from_str("db.name"), database); + meta.insert(self.from_str("out.host"), host); + metrics.insert(self.from_str("out.port"), port as f64); + + let span = Span { + start, + trace_id, + span_id, + parent_id, + span_type: self.from_str("sql"), + name: self.from_str("mysql.query"), + resource: sql, + service: self.from_str("unnamed-app-mysql"), + error: 0, + duration: 0, + meta, + metrics + }; + + self.start_span(span); + } + + fn process_finish_web_request(&mut self, _: &[Rc], mut rd: R) { read_array_len(&mut rd).unwrap(); let start = read_u64(&mut rd).unwrap(); @@ -293,7 +319,14 @@ impl Processor { trace.spans.insert(span.span_id, span); } - fn from_str(&self, s: &str) -> Rc { - self.strings.get(s).unwrap().clone() + fn from_str(&mut self, s: &str) -> Rc { + match self.strings.get(s) { + Some(s) => s.clone(), + None => { + let s: Rc = Rc::from(s); + self.strings.insert(s.clone()); + s + } + } } } From c69320ca26b5cce57fef1bf9c12fc76bcd740f75 Mon Sep 17 00:00:00 2001 From: rochdev Date: Fri, 3 Mar 2023 10:40:58 -0500 Subject: [PATCH 16/19] add ffi crate --- .../plugin-koa/internal-tracer/encoder.js | 29 +++++++++++--- .../sirun/plugin-koa/internal-tracer/now.js | 10 +++++ collector/Cargo.lock | 40 +++++++++---------- collector/Cargo.toml | 1 + collector/ffi/Cargo.toml | 16 ++++++++ collector/ffi/src/lib.rs | 34 ++++++++++++++++ collector/hyper-client/src/lib.rs | 25 ++++++++++-- package.json | 1 + yarn.lock | 11 +++++ 9 files changed, 137 insertions(+), 30 deletions(-) create mode 100644 benchmark/sirun/plugin-koa/internal-tracer/now.js create mode 100644 collector/ffi/Cargo.toml create mode 100644 collector/ffi/src/lib.rs diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js index 1bef90fc9b0..660c3d57160 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -4,10 +4,8 @@ const Chunk = require('../../../../packages/dd-trace/src/encode/chunk') const { storage } = require('../../../../packages/datadog-core') const { Client } = require('./client') const { zeroId } = require('./id') +const { now } = require('./now') -const processStartTime = BigInt(Date.now() * 1e6) -const processStartTicks = process.hrtime.bigint() -const now = () => Number(processStartTime + process.hrtime.bigint() - processStartTicks) // const service = process.env.DD_SERVICE || 'unnamed-node-app' const ARRAY_OF_TWO = 0x92 const SOFT_LIMIT = 8 * 1024 * 1024 // 8MB @@ -171,10 +169,31 @@ class Encoder { if (count === 0) return const data = this.makePayload() - const path = `/v0.1/events` this._timer = clearTimeout(this._timer) - this._client.request({ data, path, count }, done) + + if (process.env.WITH_NATIVE_COLLECTOR) { + this.flushFfi(data, done) + } else { + const path = `/v0.1/events` + this._client.request({ data, path, count }, done) + } + } + + // TODO: Use node:ffi when it lands. + // https://github.com/nodejs/node/pull/46905 + flushFfi (data, done) { + const path = require('path') + const { getNativeFunction, getBufferPointer } = require('sbffi') + const libPath = path.normalize( + path.join(__dirname, '../../../../collector/target/release/libffi.dylib') + ) + const submit = getNativeFunction(libPath, 'submit', 'uint32_t', ['uint32_t', 'uint8_t *']) + const ptr = getBufferPointer(data) + + submit(data.length, ptr) + + done() } reset () { diff --git a/benchmark/sirun/plugin-koa/internal-tracer/now.js b/benchmark/sirun/plugin-koa/internal-tracer/now.js new file mode 100644 index 00000000000..bdccbfd777f --- /dev/null +++ b/benchmark/sirun/plugin-koa/internal-tracer/now.js @@ -0,0 +1,10 @@ +'use strict' + +const processStartTime = BigInt(Date.now() * 1e6) +const processStartTicks = process.hrtime.bigint() + +function now () { + Number(processStartTime + process.hrtime.bigint() - processStartTicks) +} + +module.exports = { now } diff --git a/collector/Cargo.lock b/collector/Cargo.lock index a045ea01c84..4aff86a1c4a 100644 --- a/collector/Cargo.lock +++ b/collector/Cargo.lock @@ -51,6 +51,17 @@ dependencies = [ "rmp", ] +[[package]] +name = "ffi" +version = "0.1.0" +dependencies = [ + "common", + "hyper", + "hyper-client", + "libc", + "tokio", +] + [[package]] name = "fnv" version = "1.0.7" @@ -262,7 +273,7 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -310,7 +321,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -411,9 +422,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -432,9 +443,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -447,7 +458,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -557,21 +568,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - [[package]] name = "windows-sys" version = "0.45.0" diff --git a/collector/Cargo.toml b/collector/Cargo.toml index f2f8228c1dd..2be78b472e8 100644 --- a/collector/Cargo.toml +++ b/collector/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "common", + "ffi", "hyper-client", "server" ] diff --git a/collector/ffi/Cargo.toml b/collector/ffi/Cargo.toml new file mode 100644 index 00000000000..613376560d9 --- /dev/null +++ b/collector/ffi/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "ffi" +version = "0.1.0" +edition = "2021" + +[lib] +crate_type = ["cdylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +common = { path = "../common" } +hyper-client = { path = "../hyper-client" } +hyper = "0.14.24" +tokio = "1.25.0" +libc = "0.2.139" diff --git a/collector/ffi/src/lib.rs b/collector/ffi/src/lib.rs new file mode 100644 index 00000000000..e724b29e380 --- /dev/null +++ b/collector/ffi/src/lib.rs @@ -0,0 +1,34 @@ +use common::exporting::agent::AgentExporter; +use common::processing::Processor; +use hyper::body::Buf; +use hyper_client::HyperClient; +use tokio::sync::mpsc::{self, Receiver, Sender}; + +extern crate libc; + +#[no_mangle] +pub extern "C" fn submit(size: usize, ptr: *const u8) -> u32 { + internal_submit(unsafe { + std::slice::from_raw_parts(ptr as *const u8, size as usize) + }) as u32 +} + +#[tokio::main] +async fn internal_submit(payload: &[u8]) -> u32 { + let (tx, mut rx): (Sender<()>, Receiver<()>) = mpsc::channel(1); + + let mut client = Box::new(HyperClient::new()); + + client.on_response(tx); + + let exporter = Box::new(AgentExporter::new(client)); + let mut processor = Processor::new(exporter); + let mut rd = payload.reader(); + + processor.process(&mut rd); + processor.flush(); + + rx.recv().await.unwrap(); + + 0 // TODO: Return proper response buffer instead. +} diff --git a/collector/hyper-client/src/lib.rs b/collector/hyper-client/src/lib.rs index f87342a295b..31988c7fe6a 100644 --- a/collector/hyper-client/src/lib.rs +++ b/collector/hyper-client/src/lib.rs @@ -1,10 +1,20 @@ use common::client::Client; +use tokio::sync::mpsc::Sender; -pub struct HyperClient {} +pub struct HyperClient { + tx: Option> +} impl HyperClient { pub fn new() -> Self { - Self {} + Self { + tx: None + } + } + + // TODO: Require a sender in `new()` instead. + pub fn on_response (&mut self, tx: Sender<()>) { + self.tx = Some(tx); } } @@ -29,8 +39,17 @@ impl Client for HyperClient { .body(hyper::Body::from(data)) .unwrap(); + let tx = self.tx.clone(); + tokio::spawn(async move { - hyper::Client::new().request(req).await.unwrap(); + let res = hyper::Client::new().request(req).await.unwrap(); + + // Discard the response for now. + hyper::body::to_bytes(res.into_body()).await.unwrap(); + + if let Some(tx) = tx { + tx.send(()).await.unwrap(); + } }); } } diff --git a/package.json b/package.json index 97801cdf259..9bf4634c09c 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "path-to-regexp": "^0.1.2", "protobufjs": "^7.1.2", "retry": "^0.10.1", + "sbffi": "bengl/sbffi", "semver": "^5.5.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index b73febf9ef7..987073047a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2646,6 +2646,11 @@ node-abort-controller@^3.0.1: resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e" integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw== +node-addon-api@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.0.0.tgz#cfb3574e6df708ff71a30db6c4762d9e06e11c27" + integrity sha512-GyHvgPvUXBvAkXa0YvYnhilSB1A+FRYMpIVggKzPZqdaZfevZOuzfWzyvgzOwRLHBeo/MMswmJFsrNF4Nw1pmA== + node-gyp-build@^3.9.0: version "3.9.0" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-3.9.0.tgz#53a350187dd4d5276750da21605d1cb681d09e25" @@ -3190,6 +3195,12 @@ safe-regex-test@^1.0.0: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sbffi@bengl/sbffi: + version "1.0.4" + resolved "https://codeload.github.com/bengl/sbffi/tar.gz/35a2584cec76060b7a883f0ece5c956e92d01144" + dependencies: + node-addon-api "^6.0.0" + semver@5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" From 7d7b97a88256deb934b0d6944f875fa8f8e0c42d Mon Sep 17 00:00:00 2001 From: rochdev Date: Fri, 3 Mar 2023 11:58:58 -0500 Subject: [PATCH 17/19] handle ffi errors --- .../plugin-koa/internal-tracer/encoder.js | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js index 660c3d57160..ea7c7755d0a 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -183,17 +183,20 @@ class Encoder { // TODO: Use node:ffi when it lands. // https://github.com/nodejs/node/pull/46905 flushFfi (data, done) { - const path = require('path') - const { getNativeFunction, getBufferPointer } = require('sbffi') - const libPath = path.normalize( - path.join(__dirname, '../../../../collector/target/release/libffi.dylib') - ) - const submit = getNativeFunction(libPath, 'submit', 'uint32_t', ['uint32_t', 'uint8_t *']) - const ptr = getBufferPointer(data) - - submit(data.length, ptr) - - done() + try { + const path = require('path') + const { getNativeFunction, getBufferPointer } = require('sbffi') + const libPath = path.normalize( + path.join(__dirname, '../../../../collector/target/release/libffi.dylib') + ) + const submit = getNativeFunction(libPath, 'submit', 'uint32_t', ['uint32_t', 'uint8_t *']) + const ptr = getBufferPointer(data) + + submit(data.length, ptr) + done() + } catch (e) { + done(e) + } } reset () { From 6cbc353a11b8dc6e48cab5a56906e016336e0096 Mon Sep 17 00:00:00 2001 From: rochdev Date: Fri, 3 Mar 2023 11:59:16 -0500 Subject: [PATCH 18/19] update sirun benchmark metadata for ci --- benchmark/sirun/plugin-koa/meta.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/sirun/plugin-koa/meta.json b/benchmark/sirun/plugin-koa/meta.json index 86f7035f9bf..f69dc680fff 100644 --- a/benchmark/sirun/plugin-koa/meta.json +++ b/benchmark/sirun/plugin-koa/meta.json @@ -13,12 +13,14 @@ "with-internal-tracer": { "env": { "WITH_INTERNAL_TRACER": "true", - "DD_TRACE_AGENT_PROTOCOL_VERSION": "0.5" + "WITH_FAKE_DB": "true", + "WITH_NATIVE_COLLECTOR": "true" } }, "with-tracer": { "env": { "WITH_TRACER": "true", + "WITH_FAKE_DB": "true", "DD_TRACE_AGENT_PROTOCOL_VERSION": "0.5" } } From 3c8915782e23279b740a72355c48f21e69f57c94 Mon Sep 17 00:00:00 2001 From: rochdev Date: Sat, 4 Mar 2023 09:53:56 -0500 Subject: [PATCH 19/19] add wasm support and optimize cargo files --- benchmark/sirun/plugin-koa/agent.js | 5 +- .../plugin-koa/internal-tracer/client.js | 6 +- .../plugin-koa/internal-tracer/encoder.js | 17 ++ collector/Cargo.lock | 217 ++++++------------ collector/Cargo.toml | 3 +- collector/ffi/Cargo.toml | 3 +- collector/ffi/src/lib.rs | 3 +- collector/hyper-client/Cargo.toml | 4 +- collector/server/Cargo.toml | 4 +- collector/wasm/Cargo.toml | 13 ++ collector/wasm/src/client.rs | 35 +++ collector/wasm/src/lib.rs | 28 +++ 12 files changed, 178 insertions(+), 160 deletions(-) create mode 100644 collector/wasm/Cargo.toml create mode 100644 collector/wasm/src/client.rs create mode 100644 collector/wasm/src/lib.rs diff --git a/benchmark/sirun/plugin-koa/agent.js b/benchmark/sirun/plugin-koa/agent.js index da42f07a823..2d146a75b77 100644 --- a/benchmark/sirun/plugin-koa/agent.js +++ b/benchmark/sirun/plugin-koa/agent.js @@ -3,6 +3,7 @@ const express = require('express') const bodyParser = require('body-parser') +const port = process.env.PORT || 8126 const app = express() let requests = 0 @@ -13,10 +14,12 @@ app.use('*', (req, res) => { requests++ bytes += req.body.length + // console.log(require('msgpack-lite').decode(req.body)) + console.log(`Requests: ${requests}`) // eslint-disable-line no-console console.log(`Bytes: ${bytes}`) // eslint-disable-line no-console res.status(200).send() }) -app.listen(8126) +app.listen(port) diff --git a/benchmark/sirun/plugin-koa/internal-tracer/client.js b/benchmark/sirun/plugin-koa/internal-tracer/client.js index bea92069bbc..f35b055cfcb 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/client.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/client.js @@ -14,7 +14,8 @@ class Client { request (options, done) { if (options.count === 0) return - const url = new URL(DD_TRACE_AGENT_URL || 'http://127.0.0.1:8127') + const port = options.port || 8127 + const url = new URL(DD_TRACE_AGENT_URL || `http://127.0.0.1:${port}`) const isSecure = url.protocol === 'https:' const isUnix = url.protocol === 'unix:' const client = isSecure ? https : http @@ -36,8 +37,7 @@ class Client { 'Datadog-Meta-Lang': 'nodejs', 'Datadog-Meta-Lang-Version': process.version, 'Datadog-Meta-Lang-Interpreter': process.jsEngine || 'v8', - 'Datadog-Meta-Tracer-Version': tracerVersion, - 'X-Datadog-Trace-Count': String(options.count) + 'Datadog-Meta-Tracer-Version': tracerVersion }, timeout } diff --git a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js index ea7c7755d0a..acca446bebe 100644 --- a/benchmark/sirun/plugin-koa/internal-tracer/encoder.js +++ b/benchmark/sirun/plugin-koa/internal-tracer/encoder.js @@ -174,6 +174,8 @@ class Encoder { if (process.env.WITH_NATIVE_COLLECTOR) { this.flushFfi(data, done) + } else if (process.env.WITH_WASM_COLLECTOR) { + this.flushWasm(data, done) } else { const path = `/v0.1/events` this._client.request({ data, path, count }, done) @@ -199,6 +201,21 @@ class Encoder { } } + // In collector folder: + // cargo build -r -p wasm --target wasm32-unknown-unknown + // wasm-bindgen + // --target nodejs ./target/wasm32-unknown-unknown/release/wasm.wasm + // --out-dir=./target/wasm32-unknown-unknown/release/ + flushWasm (payload, done) { + const libPath = '../../../../collector/target/wasm32-unknown-unknown/release/wasm.js' + const { collect } = require(libPath) + + const data = collect(payload) + const path = `/v0.5/traces` + + this._client.request({ data, path, port: 8126 }, done) + } + reset () { this._reset() } diff --git a/collector/Cargo.lock b/collector/Cargo.lock index 4aff86a1c4a..fc5dc590727 100644 --- a/collector/Cargo.lock +++ b/collector/Cargo.lock @@ -20,10 +20,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] -name = "bitflags" -version = "1.3.2" +name = "bumpalo" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byteorder" @@ -47,7 +47,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" name = "common" version = "0.1.0" dependencies = [ - "hashbrown 0.13.2", + "hashbrown", "rmp", ] @@ -56,7 +56,6 @@ name = "ffi" version = "0.1.0" dependencies = [ "common", - "hyper", "hyper-client", "libc", "tokio", @@ -83,12 +82,6 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" -[[package]] -name = "futures-sink" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" - [[package]] name = "futures-task" version = "0.3.26" @@ -107,31 +100,6 @@ dependencies = [ "pin-utils", ] -[[package]] -name = "h2" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.13.2" @@ -194,7 +162,6 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", "http", "http-body", "httparse", @@ -217,21 +184,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "indexmap" -version = "1.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "libc" @@ -239,16 +196,6 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.17" @@ -258,12 +205,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - [[package]] name = "mio" version = "0.8.6" @@ -301,29 +242,6 @@ version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-sys", -] - [[package]] name = "paste" version = "1.0.11" @@ -360,15 +278,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - [[package]] name = "rmp" version = "0.8.11" @@ -380,12 +289,6 @@ dependencies = [ "paste", ] -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - [[package]] name = "server" version = "0.1.0" @@ -396,30 +299,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "slab" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - [[package]] name = "socket2" version = "0.4.9" @@ -448,14 +327,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", - "bytes", "libc", - "memchr", "mio", "num_cpus", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "socket2", "tokio-macros", "windows-sys", @@ -472,20 +347,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-util" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - [[package]] name = "tower-service" version = "0.3.2" @@ -520,9 +381,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "775c11906edafc97bc378816b94585fbd9a054eabaf86fdd0ced94af449efab7" [[package]] name = "version_check" @@ -546,6 +407,68 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm" +version = "0.1.0" +dependencies = [ + "common", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + [[package]] name = "winapi" version = "0.3.9" diff --git a/collector/Cargo.toml b/collector/Cargo.toml index 2be78b472e8..4ebff0acb1c 100644 --- a/collector/Cargo.toml +++ b/collector/Cargo.toml @@ -3,7 +3,8 @@ members = [ "common", "ffi", "hyper-client", - "server" + "server", + "wasm" ] [profile.release] diff --git a/collector/ffi/Cargo.toml b/collector/ffi/Cargo.toml index 613376560d9..f181873cb15 100644 --- a/collector/ffi/Cargo.toml +++ b/collector/ffi/Cargo.toml @@ -11,6 +11,5 @@ crate_type = ["cdylib"] [dependencies] common = { path = "../common" } hyper-client = { path = "../hyper-client" } -hyper = "0.14.24" -tokio = "1.25.0" +tokio = { version = "1.25.0", features = ["macros", "rt-multi-thread", "sync"] } libc = "0.2.139" diff --git a/collector/ffi/src/lib.rs b/collector/ffi/src/lib.rs index e724b29e380..810c2be152c 100644 --- a/collector/ffi/src/lib.rs +++ b/collector/ffi/src/lib.rs @@ -1,6 +1,5 @@ use common::exporting::agent::AgentExporter; use common::processing::Processor; -use hyper::body::Buf; use hyper_client::HyperClient; use tokio::sync::mpsc::{self, Receiver, Sender}; @@ -23,7 +22,7 @@ async fn internal_submit(payload: &[u8]) -> u32 { let exporter = Box::new(AgentExporter::new(client)); let mut processor = Processor::new(exporter); - let mut rd = payload.reader(); + let mut rd = payload; processor.process(&mut rd); processor.flush(); diff --git a/collector/hyper-client/Cargo.toml b/collector/hyper-client/Cargo.toml index b1c416ac261..319f4e73db3 100644 --- a/collector/hyper-client/Cargo.toml +++ b/collector/hyper-client/Cargo.toml @@ -7,5 +7,5 @@ edition = "2021" [dependencies] common = { path = "../common" } -hyper = { version = "0.14.24", features = ["full"] } -tokio = { version = "1.25.0", features = ["full"] } +hyper = { version = "0.14.24", features = ["client", "http1", "runtime"] } +tokio = { version = "1.25.0", features = ["rt-multi-thread", "sync"] } diff --git a/collector/server/Cargo.toml b/collector/server/Cargo.toml index 752c67997b5..2fce1df1ce6 100644 --- a/collector/server/Cargo.toml +++ b/collector/server/Cargo.toml @@ -8,5 +8,5 @@ edition = "2021" [dependencies] common = { path = "../common" } hyper-client = { path = "../hyper-client" } -hyper = "0.14.24" -tokio = "1.25.0" +hyper = { version = "0.14.24", features = ["http1", "runtime", "server"] } +tokio = { version = "1.25.0", features = ["macros", "rt-multi-thread", "sync"] } diff --git a/collector/wasm/Cargo.toml b/collector/wasm/Cargo.toml new file mode 100644 index 00000000000..8b566e70e73 --- /dev/null +++ b/collector/wasm/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "wasm" +version = "0.1.0" +edition = "2021" + +[lib] +crate_type = ["cdylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +common = { path = "../common" } +wasm-bindgen = "0.2.84" diff --git a/collector/wasm/src/client.rs b/collector/wasm/src/client.rs new file mode 100644 index 00000000000..00148c5a1b5 --- /dev/null +++ b/collector/wasm/src/client.rs @@ -0,0 +1,35 @@ +use common::client::Client; +use std::sync::mpsc::SyncSender; + +pub struct BufferClient { + tx: Option>> +} + +impl BufferClient { + pub fn new() -> Self { + Self { + tx: None + } + } + + // TODO: Require a sender in `new()` instead. + pub fn on_response (&mut self, tx: SyncSender>) { + self.tx = Some(tx); + } +} + +impl Default for BufferClient { + fn default() -> Self { + Self::new() + } +} + +impl Client for BufferClient { + fn request(&self, data: Vec) { + let tx = self.tx.clone(); + + if let Some(tx) = tx { + tx.send(data).unwrap(); + } + } +} diff --git a/collector/wasm/src/lib.rs b/collector/wasm/src/lib.rs new file mode 100644 index 00000000000..d294672d24b --- /dev/null +++ b/collector/wasm/src/lib.rs @@ -0,0 +1,28 @@ + +use client::BufferClient; +use common::exporting::agent::AgentExporter; +use common::processing::Processor; +use std::sync::mpsc::{SyncSender, Receiver, self}; +use wasm_bindgen::prelude::*; + +pub mod client; + +// TODO: Use WASI to submit from here instead of just returning the data. + +#[wasm_bindgen] +pub fn collect(payload: &[u8]) -> Vec { + let (tx, rx): (SyncSender>, Receiver>) = mpsc::sync_channel(1); + + let mut client = Box::new(BufferClient::new()); + + client.on_response(tx); + + let exporter = Box::new(AgentExporter::new(client)); + let mut processor = Processor::new(exporter); + let mut rd = payload; + + processor.process(&mut rd); + processor.flush(); + + rx.recv().unwrap() +}