diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json new file mode 100644 index 00000000000..5d279f0909f --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -0,0 +1,108 @@ +{ + "pinned": [ + "ENTER_PACKAGE_NAME_HERE" + ], + "latests": { + "aerospike": "5.13.2", + "amqp10": "3.6.0", + "amqplib": "0.10.4", + "apollo-server-core": "3.13.0", + "@apollo/server": "4.11.2", + "@apollo/gateway": "2.9.3", + "avsc": "5.7.7", + "@smithy/smithy-client": "3.4.4", + "@aws-sdk/smithy-client": "3.374.0", + "aws-sdk": "2.1692.0", + "@azure/functions": "4.6.0", + "bluebird": "3.7.2", + "body-parser": "1.20.3", + "bunyan": "1.8.15", + "cassandra-driver": "4.7.2", + "connect": "3.7.0", + "cookie-parser": "1.4.7", + "cookie": "1.0.1", + "couchbase": "4.4.3", + "@cucumber/cucumber": "11.1.0", + "cypress": "13.15.2", + "@elastic/transport": "8.9.1", + "@elastic/elasticsearch": "8.16.1", + "elasticsearch": "16.7.3", + "express-mongo-sanitize": "2.2.0", + "express": "4.21.1", + "fastify": "5.1.0", + "find-my-way": "9.1.0", + "fs": "0.0.1-security", + "generic-pool": "3.9.0", + "@google-cloud/pubsub": "4.9.0", + "@graphql-tools/executor": "1.3.3", + "graphql": "16.9.0", + "@grpc/grpc-js": "1.12.2", + "handlebars": "4.7.8", + "@hapi/hapi": "21.3.12", + "hapi": "18.1.0", + "ioredis": "5.4.1", + "jest-environment-node": "29.7.0", + "jest-environment-jsdom": "29.7.0", + "@jest/core": "29.7.0", + "@jest/test-sequencer": "29.7.0", + "@jest/reporters": "29.7.0", + "jest-circus": "29.7.0", + "@jest/transform": "29.7.0", + "jest-config": "29.7.0", + "jest-runtime": "29.7.0", + "jest-worker": "29.7.0", + "kafkajs": "2.2.4", + "knex": "3.1.0", + "koa": "2.15.3", + "@koa/router": "13.1.0", + "koa-router": "13.0.1", + "ldapjs": "3.0.7", + "limitd-client": "2.14.1", + "lodash": "4.17.21", + "mariadb": "3.4.0", + "memcached": "2.2.2", + "microgateway-core": "3.3.4", + "moleculer": "0.14.35", + "mongodb-core": "3.2.7", + "mongodb": "6.10.0", + "mongoose": "8.8.2", + "mquery": "5.0.0", + "multer": "1.4.5-lts.1", + "mysql": "2.18.1", + "mysql2": "3.11.4", + "next": "15.0.3", + "nyc": "17.1.0", + "openai": "4.72.0", + "@opensearch-project/opensearch": "2.13.0", + "oracledb": "6.7.0", + "paperplane": "3.1.2", + "passport-http": "0.3.0", + "passport-local": "1.0.0", + "pg": "8.13.1", + "pino": "9.5.0", + "pino-pretty": "13.0.0", + "@playwright/test": "1.49.0", + "playwright": "1.49.0", + "promise-js": "0.0.7", + "promise": "8.3.0", + "protobufjs": "7.4.0", + "pug": "3.0.3", + "q": "1.5.1", + "qs": "6.13.1", + "@node-redis/client": "1.0.6", + "@redis/client": "1.6.0", + "redis": "4.7.0", + "restify": "11.1.0", + "rhea": "3.0.3", + "router": "1.3.8", + "selenium-webdriver": "4.26.0", + "sequelize": "6.37.5", + "sharedb": "5.1.1", + "tedious": "18.6.1", + "undici": "6.21.0", + "vitest": "2.1.5", + "@vitest/runner": "2.1.5", + "when": "3.7.8", + "winston": "3.17.0" + } +} \ No newline at end of file diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index 2f2ef2c1cd1..85191002004 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -16,6 +16,7 @@ const { const hooks = require('./hooks') const instrumentations = require('./instrumentations') +const latests = require('./latests.json') const names = Object.keys(hooks) const pathSepExpr = new RegExp(`\\${path.sep}`, 'g') const disabledInstrumentations = new Set( @@ -110,7 +111,7 @@ for (const packageName of names) { namesAndSuccesses[`${name}@${version}`] = false } - if (matchVersion(version, versions)) { + if (matchVersion(version, versions) && matchesLatestSupported(name, version)) { // Check if the hook already has a set moduleExport if (hook[HOOK_SYMBOL].has(moduleExports)) { namesAndSuccesses[`${name}@${version}`] = true @@ -158,6 +159,17 @@ function matchVersion (version, ranges) { return !version || (ranges && ranges.some(range => satisfies(version, range))) } +function matchesLatestSupported (name, version) { + if (latests.pinned.includes(name)) { + // These ones are deliberately pinned to a specific version. That + // means we can skip this check, since it will already have been checked + // to be lower than latest. + return true + } + const latest = latests.latests[name] + return matchVersion(version, ['<=' + latest]) +} + function getVersion (moduleBaseDir) { if (moduleBaseDir) { return requirePackageJson(moduleBaseDir, module).version diff --git a/packages/dd-trace/test/setup/mocha.js b/packages/dd-trace/test/setup/mocha.js index 2abf0b86c3b..232ff85d82e 100644 --- a/packages/dd-trace/test/setup/mocha.js +++ b/packages/dd-trace/test/setup/mocha.js @@ -13,6 +13,8 @@ const { storage } = require('../../../datadog-core') const { schemaDefinitions } = require('../../src/service-naming/schemas') const { getInstrumentation } = require('./helpers/load-inst') +const latestVersions = require('../../../datadog-instrumentations/src/helpers/latests.json').latests + global.withVersions = withVersions global.withExports = withExports global.withNamingSchema = withNamingSchema @@ -142,12 +144,28 @@ function withPeerService (tracer, pluginName, spanGenerationFn, service, service }) } +function isVersionInRange (version, latestVersion) { + if (!latestVersion) return true + try { + return semver.lte(version, latestVersion) + } catch (e) { + return true // Safety fallback for invalid semver strings + } +} + function withVersions (plugin, modules, range, cb) { - const instrumentations = typeof plugin === 'string' ? getInstrumentation(plugin) : [].concat(plugin) + // Normalize plugin parameter to an array of instrumentation objects + const instrumentations = typeof plugin === 'string' + ? getInstrumentation(plugin) + : [].concat(plugin) + + // Extract all plugin names from instrumentations const names = instrumentations.map(instrumentation => instrumentation.name) + // Ensure modules is an array modules = [].concat(modules) + // Add dependent instrumentations for external plugins names.forEach(name => { if (externals[name]) { [].concat(externals[name]).forEach(external => { @@ -156,46 +174,107 @@ function withVersions (plugin, modules, range, cb) { } }) + // Handle case where range is omitted if (!cb) { cb = range range = null } + // Process each module modules.forEach(moduleName => { + // Skip if not in the PACKAGE_NAMES env var filter (when specified) if (process.env.PACKAGE_NAMES) { const packages = process.env.PACKAGE_NAMES.split(',') - if (!packages.includes(moduleName)) return } + // Map to store unique versions to test const testVersions = new Map() + // Collect versions to test from applicable instrumentations instrumentations .filter(instrumentation => instrumentation.name === moduleName) .forEach(instrumentation => { + // Use version range from environment or from instrumentation const versions = process.env.PACKAGE_VERSION_RANGE ? [process.env.PACKAGE_VERSION_RANGE] : instrumentation.versions + + // Process each version/range that passes the RANGE filter (if set) versions .filter(version => !process.env.RANGE || semver.subset(version, process.env.RANGE)) .forEach(version => { + // Handle exact version specifications (not wildcards) if (version !== '*') { - const min = semver.coerce(version).version - - testVersions.set(min, { range: version, test: min }) + // Handle explicit minimum versions in ranges like ">=2.0.0" + if (version.startsWith('>=')) { + const minVersion = version.substring(2).trim() + const parsedMinVersion = semver.valid(minVersion) + ? minVersion + : semver.coerce(minVersion).version + testVersions.set(parsedMinVersion, { range: version, test: parsedMinVersion }) + } else { + // For other version specs, coerce to a standard semver format + const min = semver.coerce(version).version + testVersions.set(min, { range: version, test: min }) + } } - const max = require(`../../../../versions/${moduleName}@${version}`).version() - - testVersions.set(max, { range: version, test: version }) + // Try to find the latest compatible version from latests.json + if (latestVersions[moduleName] && !process.env.PACKAGE_VERSION_RANGE) { + // For exact versions + if (semver.valid(version)) { + // Use specified version if it's newer than latest, otherwise use latest + const testVersion = isVersionInRange(version, latestVersions[moduleName]) + ? version + : latestVersions[moduleName] + testVersions.set(testVersion, { range: version, test: testVersion }) + } else if (semver.validRange(version)) { // For version ranges + // Find the highest version that satisfies the range + const testVersion = semver.maxSatisfying([latestVersions[moduleName]], version) + if (testVersion) { + testVersions.set(testVersion, { range: version, test: testVersion }) + } + } + } else if (latestVersions[moduleName]) { // When PACKAGE_VERSION_RANGE is specified + const range = process.env.PACKAGE_VERSION_RANGE + // Check if latest version satisfies the range, or find max version that does + const testVersion = semver.satisfies(latestVersions[moduleName], range) + ? latestVersions[moduleName] + : semver.maxSatisfying([latestVersions[moduleName]], range) + if (testVersion) { + testVersions.set(testVersion, { range: version, test: testVersion }) + } + } else { // Fallback method: try to load version from the filesystem + try { + // Try to dynamically require the version module + const max = require(`../../../../versions/${moduleName}@${version}`).version() + testVersions.set(max, { range: version, test: version }) + } catch (err) { + // FIX ME: log + // Try an alternate path with a coerced version string + try { + const coercedVersion = semver.coerce(version).version + const max = require(`../../../../versions/${moduleName}@${coercedVersion}`).version() + testVersions.set(max, { range: version, test: coercedVersion }) + } catch (innerErr) { + // FIX ME: log + } + } + } }) }) + // Create test suites for each version Array.from(testVersions) + // Filter by the specified range if provided .filter(v => !range || semver.satisfies(v[0], range)) - .sort(v => v[0].localeCompare(v[0])) + // Sort by semver to run tests in version order + .sort((a, b) => semver.compare(a[0], b[0])) + // Format the version objects .map(v => Object.assign({}, v[1], { version: v[0] })) .forEach(v => { + // Resolve the path to the module's node_modules directory const versionPath = path.resolve( __dirname, '../../../../versions/', `${moduleName}@${v.test}/node_modules` @@ -216,13 +295,14 @@ function withVersions (plugin, modules, range, cb) { process.env.NODE_PATH = [process.env.NODE_PATH, versionPath] .filter(x => x && x !== 'undefined') .join(os.platform() === 'win32' ? ';' : ':') - require('module').Module._initPaths() }) + // Run the provided test callback with the version information cb(v.test, moduleName, v.version) after(() => { + // Restore the original NODE_PATH process.env.NODE_PATH = nodePath require('module').Module._initPaths() }) diff --git a/scripts/helpers/versioning.js b/scripts/helpers/versioning.js new file mode 100644 index 00000000000..b51eaafee8b --- /dev/null +++ b/scripts/helpers/versioning.js @@ -0,0 +1,78 @@ +const fs = require('fs') +const path = require('path') +const childProcess = require('child_process') +const proxyquire = require('proxyquire') + +const versionLists = {} +const names = [] + +const filter = process.env.hasOwnProperty('PLUGINS') && process.env.PLUGINS.split('|') + +fs.readdirSync(path.join(__dirname, '../../packages/datadog-instrumentations/src')) + .filter(file => file.endsWith('js')) + .forEach(file => { + file = file.replace('.js', '') + + if (!filter || filter.includes(file)) { + names.push(file) + } + }) + +async function getVersionList (name) { + if (versionLists[name]) { + return versionLists[name] + } + const list = await npmView(`${name} versions`) + versionLists[name] = list + return list +} + +function npmView (input) { + return new Promise((resolve, reject) => { + childProcess.exec(`npm view ${input} --json`, (err, stdout) => { + if (err) { + reject(err) + return + } + resolve(JSON.parse(stdout.toString('utf8'))) + }) + }) +} + +function loadInstFile (file, instrumentations) { + const instrument = { + addHook (instrumentation) { + instrumentations.push(instrumentation) + } + } + + const instPath = path.join(__dirname, `../../packages/datadog-instrumentations/src/${file}`) + + proxyquire.noPreserveCache()(instPath, { + './helpers/instrument': instrument, + '../helpers/instrument': instrument + }) +} + +function getInternals () { + return names.map(key => { + const instrumentations = [] + const name = key + + try { + loadInstFile(`${name}/server.js`, instrumentations) + loadInstFile(`${name}/client.js`, instrumentations) + } catch (e) { + loadInstFile(`${name}.js`, instrumentations) + } + + return instrumentations + }).reduce((prev, next) => prev.concat(next), []) +} + +module.exports = { + getVersionList, + npmView, + loadInstFile, + getInternals +} diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index 212dc5928ed..b93c752df84 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -6,9 +6,13 @@ const path = require('path') const crypto = require('crypto') const semver = require('semver') const exec = require('./helpers/exec') -const childProcess = require('child_process') const externals = require('../packages/dd-trace/test/plugins/externals') const { getInstrumentation } = require('../packages/dd-trace/test/setup/helpers/load-inst') +const { + getVersionList, + npmView +} = require('./helpers/versioning') +const latests = require('../packages/datadog-instrumentations/src/helpers/latests.json') const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json') @@ -16,7 +20,6 @@ const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require // Can remove couchbase after removing support for couchbase <= 3.2.0 const excludeList = os.arch() === 'arm64' ? ['aerospike', 'couchbase', 'grpc', 'oracledb'] : [] const workspaces = new Set() -const versionLists = {} const deps = {} const filter = process.env.hasOwnProperty('PLUGINS') && process.env.PLUGINS.split('|') @@ -63,17 +66,55 @@ async function assertVersions () { } async function assertInstrumentation (instrumentation, external) { + const name = instrumentation.name const versions = process.env.PACKAGE_VERSION_RANGE && !external ? [process.env.PACKAGE_VERSION_RANGE] : [].concat(instrumentation.versions || []) - for (const version of versions) { - if (version) { - if (version !== '*') { - await assertModules(instrumentation.name, semver.coerce(version).version, external) + for (const versionRange of versions) { + if (!versionRange || versionRange === '*') continue + + // For exact versions, just use them + if (semver.valid(versionRange)) { + await assertModules(name, versionRange, external) + continue + } + // For version ranges + if (semver.validRange(versionRange)) { + const latestVersion = latests.latests[name] + + // Always install the latest version from latests.json if it satisfies the range + if (latestVersion && semver.satisfies(latestVersion, versionRange)) { + await assertModules(name, latestVersion, external) } - await assertModules(instrumentation.name, version, external) + // For ranges with a minimum version (like >=2.0.0), also install the minimum version + if (versionRange.startsWith('>=')) { + // Extract the exact version after >= + const minVersion = versionRange.substring(2).trim() + + // If it's not a valid semver (like just "2"), coerce it to a proper version (2.0.0) + const parsedMinVersion = semver.valid(minVersion) ? minVersion : semver.coerce(minVersion).version + + if (parsedMinVersion && parsedMinVersion !== latestVersion) { + await assertModules(name, parsedMinVersion, external) + } + } + + // For broader ranges, get the lower bound + if (!versionRange.startsWith('>=')) { + let lowerBound + try { + // Try to extract the lower bound from the range + lowerBound = semver.coerce(versionRange).version + } catch (e) { + // FIX ME: log? + } + + if (lowerBound && lowerBound !== latestVersion) { + await assertModules(name, lowerBound, external) + } + } } } } @@ -133,7 +174,17 @@ async function assertPackage (name, version, dependencyVersionRange, external) { } async function addDependencies (dependencies, name, versionRange) { - const versionList = await getVersionList(name) + let versionList = await getVersionList(name) + if (!latests.pinned.includes(name)) { + const maxVersion = latests.latests[name] + versionList = versionList.map(version => { + if (version.startsWith('>=') && !version.includes('<')) { + return version + ' <=' + maxVersion + } else { + return version + } + }) + } const version = semver.maxSatisfying(versionList, versionRange) const pkgJson = await npmView(`${name}@${version}`) for (const dep of deps[name]) { @@ -152,27 +203,6 @@ async function addDependencies (dependencies, name, versionRange) { } } -async function getVersionList (name) { - if (versionLists[name]) { - return versionLists[name] - } - const list = await npmView(`${name} versions`) - versionLists[name] = list - return list -} - -function npmView (input) { - return new Promise((resolve, reject) => { - childProcess.exec(`npm view ${input} --json`, (err, stdout) => { - if (err) { - reject(err) - return - } - resolve(JSON.parse(stdout.toString('utf8'))) - }) - }) -} - function assertIndex (name, version) { const index = `'use strict' @@ -208,7 +238,15 @@ function install () { } function addFolder (name, version) { - const basename = [name, version].filter(val => val).join('@') + // Skip if either name or version is undefined + // was seeing many of these in the logs when debugging + // e.g. mysql@undefined + if (!name || !version) { + return + } + + const basename = `${name}@${version}` + if (!excludeList.includes(name)) workspaces.add(basename) } diff --git a/scripts/outdated.js b/scripts/outdated.js new file mode 100644 index 00000000000..72cf0fcfd4f --- /dev/null +++ b/scripts/outdated.js @@ -0,0 +1,129 @@ +/* eslint-disable no-console */ +const { + getInternals, + npmView +} = require('./helpers/versioning') +const path = require('path') +const fs = require('fs') + +const latestsPath = path.join( + __dirname, + '..', + 'packages', + 'datadog-instrumentations', + 'src', + 'helpers', + 'latests.json' +) + +// Get internal package names from existing getInternals helper +const internalsNames = Array.from(new Set(getInternals().map(n => n.name))) + .filter(x => typeof x === 'string' && x !== 'child_process' && !x.startsWith('node:')) + +// Initial structure with placeholder for pinned packages +const initialStructure = { + pinned: ['ENTER_PACKAGE_NAME_HERE'], + latests: {} +} + +/** + * Updates latests.json with the current latest versions from npm + */ +async function fix () { + console.log('Starting fix operation...') + console.log(`Found ${internalsNames.length} packages to process`) + + let outputData = initialStructure + if (fs.existsSync(latestsPath)) { + console.log('Found existing latests.json, loading it...') + outputData = require(latestsPath) + } + + const latests = {} + let processed = 0 + const total = internalsNames.length + + for (const name of internalsNames) { + processed++ + process.stdout.write(`Processing package ${processed}/${total}: ${name}...`) + + try { + const distTags = await npmView(name + ' dist-tags') + const latest = distTags.latest + if (latest) { + latests[name] = latest + process.stdout.write(` found version ${latest}\n`) + } else { + process.stdout.write(' WARNING: no version found\n') + console.log(`Warning: Could not fetch latest version for "${name}"`) + } + } catch (error) { + process.stdout.write(' ERROR\n') + console.error(`Error fetching version for "${name}":`, error.message) + } + } + + outputData.latests = latests + console.log('\nWriting updated versions to latests.json...') + fs.writeFileSync(latestsPath, JSON.stringify(outputData, null, 2)) + console.log('Successfully updated latests.json') + console.log(`Processed ${total} packages`) +} + +/** + * Checks if latests.json matches current npm versions + */ +async function check () { + console.log('Starting version check...') + + if (!fs.existsSync(latestsPath)) { + console.log('latests.json does not exist. Run with "fix" to create it.') + process.exitCode = 1 + return + } + + const currentData = require(latestsPath) + console.log(`Found ${internalsNames.length} packages to check`) + + let processed = 0 + let mismatches = 0 + const total = internalsNames.length + + for (const name of internalsNames) { + processed++ + process.stdout.write(`Checking package ${processed}/${total}: ${name}...`) + + const latest = currentData.latests[name] + if (!latest) { + process.stdout.write(' MISSING\n') + console.log(`No latest version found for "${name}"`) + process.exitCode = 1 + continue + } + + try { + const distTags = await npmView(name + ' dist-tags') + const npmLatest = distTags.latest + if (npmLatest !== latest) { + process.stdout.write(' MISMATCH\n') + console.log(`"latests.json: is not up to date for "${name}": expected "${npmLatest}", got "${latest}"`) + process.exitCode = 1 + mismatches++ + } else { + process.stdout.write(' OK\n') + } + } catch (error) { + process.stdout.write(' ERROR\n') + console.error(`Error checking version for "${name}":`, error.message) + } + } + + console.log('\nCheck completed:') + console.log(`- Total packages checked: ${total}`) + console.log(`- Version mismatches found: ${mismatches}`) + if (mismatches > 0) { + console.log('Run with "fix" to update versions') + } +} +if (process.argv.includes('fix')) fix() +else check()