From 4f34cf26b0a663d7ce7ae6439f426017059ace83 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 28 Feb 2025 14:51:33 -0500 Subject: [PATCH 01/15] Implement basic CI version capper --- .../src/helpers/latests.json | 112 +++++++++++++++ scripts/helpers/versioning.js | 78 +++++++++++ scripts/install_plugin_modules.js | 17 ++- scripts/outdated.js | 129 ++++++++++++++++++ 4 files changed, 334 insertions(+), 2 deletions(-) create mode 100644 packages/datadog-instrumentations/src/helpers/latests.json create mode 100644 scripts/helpers/versioning.js create mode 100644 scripts/outdated.js diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json new file mode 100644 index 00000000000..9a6c82b4f8a --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -0,0 +1,112 @@ +{ + "pinned": [ + "ENTER_PACKAGE_NAME_HERE" + ], + "latests": { + "aerospike": "6.0.2", + "amqp10": "3.6.0", + "amqplib": "0.10.5", + "apollo-server-core": "3.13.0", + "@apollo/server": "4.11.3", + "@apollo/gateway": "2.10.0", + "avsc": "5.7.7", + "@smithy/smithy-client": "4.1.6", + "@aws-sdk/smithy-client": "3.374.0", + "aws-sdk": "2.1692.0", + "@azure/functions": "4.6.1", + "bluebird": "3.7.2", + "body-parser": "1.20.3", + "bunyan": "1.8.15", + "cassandra-driver": "4.8.0", + "connect": "3.7.0", + "cookie-parser": "1.4.7", + "cookie": "1.0.2", + "couchbase": "4.4.5", + "@cucumber/cucumber": "11.2.0", + "cypress": "14.1.0", + "@elastic/transport": "8.9.4", + "@elastic/elasticsearch": "8.17.1", + "elasticsearch": "16.7.3", + "express-mongo-sanitize": "2.2.0", + "express-session": "1.18.1", + "express": "4.21.2", + "fastify": "5.2.1", + "find-my-way": "9.2.0", + "fs": "0.0.1-security", + "generic-pool": "3.9.0", + "@google-cloud/pubsub": "4.10.0", + "@graphql-tools/executor": "1.4.2", + "graphql": "16.10.0", + "@grpc/grpc-js": "1.12.6", + "handlebars": "4.7.8", + "@hapi/hapi": "21.3.12", + "hapi": "18.1.0", + "ioredis": "5.5.0", + "jest-environment-node": "29.7.0", + "jest-environment-jsdom": "29.7.0", + "@jest/core": "29.7.0", + "@jest/test-sequencer": "29.7.0", + "@jest/reporters": "29.7.0", + "jest-circus": "29.7.0", + "@jest/transform": "29.7.0", + "jest-config": "29.7.0", + "jest-runtime": "29.7.0", + "jest-worker": "29.7.0", + "kafkajs": "2.2.4", + "knex": "3.1.0", + "koa": "2.16.0", + "@koa/router": "13.1.0", + "koa-router": "13.0.1", + "@langchain/core": "0.3.41", + "@langchain/openai": "0.4.4", + "ldapjs": "3.0.7", + "limitd-client": "2.14.1", + "lodash": "4.17.21", + "mariadb": "3.4.0", + "memcached": "2.2.2", + "microgateway-core": "3.3.5", + "moleculer": "0.14.35", + "mongodb-core": "3.2.7", + "mongodb": "6.14.0", + "mongoose": "8.11.0", + "mquery": "5.0.0", + "multer": "1.4.5-lts.1", + "mysql": "2.18.1", + "mysql2": "3.12.0", + "next": "15.2.0", + "node-serialize": "0.0.4", + "nyc": "17.1.0", + "openai": "4.86.1", + "@opensearch-project/opensearch": "3.4.0", + "oracledb": "6.8.0", + "paperplane": "3.1.2", + "passport-http": "0.3.0", + "passport-local": "1.0.0", + "passport": "0.7.0", + "pg": "8.13.3", + "pino": "9.6.0", + "pino-pretty": "13.0.0", + "@playwright/test": "1.50.1", + "playwright": "1.50.1", + "promise-js": "0.0.7", + "promise": "8.3.0", + "protobufjs": "7.4.0", + "pug": "3.0.3", + "q": "1.5.1", + "@node-redis/client": "1.0.6", + "@redis/client": "1.6.0", + "redis": "4.7.0", + "restify": "11.1.0", + "rhea": "3.0.3", + "router": "1.3.8", + "selenium-webdriver": "4.29.0", + "sequelize": "6.37.5", + "sharedb": "5.1.1", + "tedious": "18.6.1", + "undici": "7.4.0", + "vitest": "3.0.7", + "@vitest/runner": "3.0.7", + "when": "3.7.8", + "winston": "3.17.0" + } +} \ No newline at end of file diff --git a/scripts/helpers/versioning.js b/scripts/helpers/versioning.js new file mode 100644 index 00000000000..69cdec41df7 --- /dev/null +++ b/scripts/helpers/versioning.js @@ -0,0 +1,78 @@ +const fs = require('fs') +const path = require('path') +const childProcess = require('child_process') +const proxyquire = require('proxyquire') + +const versionLists = {} +const names = [] + +const filter = process.env.hasOwnProperty('PLUGINS') && process.env.PLUGINS.split('|') + +fs.readdirSync(path.join(__dirname, '../../packages/datadog-instrumentations/src')) + .filter(file => file.endsWith('js')) + .forEach(file => { + file = file.replace('.js', '') + + if (!filter || filter.includes(file)) { + names.push(file) + } + }) + +async function getVersionList (name) { + if (versionLists[name]) { + return versionLists[name] + } + const list = await npmView(`${name} versions`) + versionLists[name] = list + return list +} + +function npmView (input) { + return new Promise((resolve, reject) => { + childProcess.exec(`npm view ${input} --json`, (err, stdout) => { + if (err) { + reject(err) + return + } + resolve(JSON.parse(stdout.toString('utf8'))) + }) + }) +} + +function loadInstFile (file, instrumentations) { + const instrument = { + addHook (instrumentation) { + instrumentations.push(instrumentation) + } + } + + const instPath = path.join(__dirname, `../../packages/datadog-instrumentations/src/${file}`) + + proxyquire.noPreserveCache()(instPath, { + './helpers/instrument': instrument, + '../helpers/instrument': instrument + }) +} + +function getInternals () { + return names.map(key => { + const instrumentations = [] + const name = key + + try { + loadInstFile(`${name}/server.js`, instrumentations) + loadInstFile(`${name}/client.js`, instrumentations) + } catch (e) { + loadInstFile(`${name}.js`, instrumentations) + } + + return instrumentations + }).reduce((prev, next) => prev.concat(next), []) +} + +module.exports = { + getVersionList, + npmView, + loadInstFile, + getInternals +} \ No newline at end of file diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index cfcc9b29371..be951aa042e 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -9,6 +9,7 @@ const exec = require('./helpers/exec') const childProcess = require('child_process') const externals = require('../packages/dd-trace/test/plugins/externals') const { getInstrumentation } = require('../packages/dd-trace/test/setup/helpers/load-inst') +const latests = require('../packages/datadog-instrumentations/src/helpers/latests.json') const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json') @@ -106,9 +107,21 @@ function assertFolder (name, version) { } async function assertPackage (name, version, dependencyVersionRange, external) { - const dependencies = { [name]: dependencyVersionRange } + // Apply version cap from latests.json if available + // TODO: pinned versions? + let cappedVersionRange = dependencyVersionRange + if (latests.latests[name]) { + const latestVersion = latests.latests[name] + // If the range would allow versions beyond what we've tested, cap it + if (semver.validRange(dependencyVersionRange) && + !semver.subset(`<=${latestVersion}`, dependencyVersionRange)) { + cappedVersionRange = `${dependencyVersionRange} <=${latestVersion}` + } + } + + const dependencies = { [name]: cappedVersionRange } if (deps[name]) { - await addDependencies(dependencies, name, dependencyVersionRange) + await addDependencies(dependencies, name, cappedVersionRange) } const pkg = { name: [name, sha1(name).substr(0, 8), sha1(version)].filter(val => val).join('-'), diff --git a/scripts/outdated.js b/scripts/outdated.js new file mode 100644 index 00000000000..48c3a2c2755 --- /dev/null +++ b/scripts/outdated.js @@ -0,0 +1,129 @@ +/* eslint-disable no-console */ +const { + getInternals, + npmView + } = require('./helpers/versioning') + const path = require('path') + const fs = require('fs') + + const latestsPath = path.join( + __dirname, + '..', + 'packages', + 'datadog-instrumentations', + 'src', + 'helpers', + 'latests.json' + ) + + // Get internal package names from existing getInternals helper + const internalsNames = Array.from(new Set(getInternals().map(n => n.name))) + .filter(x => typeof x === 'string' && x !== 'child_process' && !x.startsWith('node:')) + + // Initial structure with placeholder for pinned packages + const initialStructure = { + pinned: ['ENTER_PACKAGE_NAME_HERE'], + latests: {} + } + + /** + * Updates latests.json with the current latest versions from npm + */ + async function fix () { + console.log('Starting fix operation...') + console.log(`Found ${internalsNames.length} packages to process`) + + let outputData = initialStructure + if (fs.existsSync(latestsPath)) { + console.log('Found existing latests.json, loading it...') + outputData = require(latestsPath) + } + + const latests = {} + let processed = 0 + const total = internalsNames.length + + for (const name of internalsNames) { + processed++ + process.stdout.write(`Processing package ${processed}/${total}: ${name}...`) + + try { + const distTags = await npmView(name + ' dist-tags') + const latest = distTags.latest + if (latest) { + latests[name] = latest + process.stdout.write(` found version ${latest}\n`) + } else { + process.stdout.write(' WARNING: no version found\n') + console.log(`Warning: Could not fetch latest version for "${name}"`) + } + } catch (error) { + process.stdout.write(' ERROR\n') + console.error(`Error fetching version for "${name}":`, error.message) + } + } + + outputData.latests = latests + console.log('\nWriting updated versions to latests.json...') + fs.writeFileSync(latestsPath, JSON.stringify(outputData, null, 2)) + console.log('Successfully updated latests.json') + console.log(`Processed ${total} packages`) + } + + /** + * Checks if latests.json matches current npm versions + */ + async function check () { + console.log('Starting version check...') + + if (!fs.existsSync(latestsPath)) { + console.log('latests.json does not exist. Run with "fix" to create it.') + process.exitCode = 1 + return + } + + const currentData = require(latestsPath) + console.log(`Found ${internalsNames.length} packages to check`) + + let processed = 0 + let mismatches = 0 + const total = internalsNames.length + + for (const name of internalsNames) { + processed++ + process.stdout.write(`Checking package ${processed}/${total}: ${name}...`) + + const latest = currentData.latests[name] + if (!latest) { + process.stdout.write(' MISSING\n') + console.log(`No latest version found for "${name}"`) + process.exitCode = 1 + continue + } + + try { + const distTags = await npmView(name + ' dist-tags') + const npmLatest = distTags.latest + if (npmLatest !== latest) { + process.stdout.write(' MISMATCH\n') + console.log(`"latests.json: is not up to date for "${name}": expected "${npmLatest}", got "${latest}"`) + process.exitCode = 1 + mismatches++ + } else { + process.stdout.write(' OK\n') + } + } catch (error) { + process.stdout.write(' ERROR\n') + console.error(`Error checking version for "${name}":`, error.message) + } + } + + console.log('\nCheck completed:') + console.log(`- Total packages checked: ${total}`) + console.log(`- Version mismatches found: ${mismatches}`) + if (mismatches > 0) { + console.log('Run with "fix" to update versions') + } + } + if (process.argv.includes('fix')) fix() + else check() \ No newline at end of file From 79293756bec4f07be8c5bed5d72330e8c6f87518 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Tue, 4 Mar 2025 12:06:11 -0500 Subject: [PATCH 02/15] Attempt to handle ranges better when capping --- scripts/install_plugin_modules.js | 104 ++++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 6 deletions(-) diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index be951aa042e..fbb918f7c1e 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -106,19 +106,111 @@ function assertFolder (name, version) { } } +// Helper function to apply version caps in a more readable way +function applyCap (versionRange, latestVersion) { + // Handle hyphen ranges (e.g., "24.8.0 - 24.9.0") + const hyphenRangeMatch = versionRange.match(/^(\d+\.\d+\.\d+)\s*-\s*(\d+\.\d+\.\d+)(.*)$/) + if (hyphenRangeMatch) { + return handleHyphenRange(hyphenRangeMatch, latestVersion) + } + + // Handle exact versions (e.g., "24.8.0") + if (semver.valid(versionRange)) { + return handleExactVersion(versionRange, latestVersion) + } + + // Handle other valid semver ranges + if (semver.validRange(versionRange)) { + return handleValidRange(versionRange, latestVersion) + } + + // If nothing else matched, return the original range + return versionRange +} + +// Handle hyphen ranges like "24.8.0 - 24.9.0" +function handleHyphenRange (match, latestVersion) { + const [, lowerBound, upperBound, extraConstraints] = match + + // Cap at the lower of: original upper bound or latest version + const effectiveUpper = semver.lt(upperBound, latestVersion) ? upperBound : latestVersion + + // Create properly formatted range + let result = `>=${lowerBound} <=${effectiveUpper}` + + // Add any extra constraints if they exist and would create a valid range + if (extraConstraints && extraConstraints.trim()) { + const combinedRange = `${result} ${extraConstraints.trim()}` + if (semver.validRange(combinedRange)) { + result = combinedRange + } + } + + return result +} + +// Handle exact versions like "24.8.0" +function handleExactVersion (version, latestVersion) { + const exactVersion = semver.clean(version) + + // If exact version is too high, cap it + if (semver.gt(exactVersion, latestVersion)) { + return latestVersion + } + + // Otherwise keep exact version with cap + return `${exactVersion} <=${latestVersion}` +} + +// Handle general semver ranges +function handleValidRange (range, latestVersion) { + // Only apply cap if necessary + if (semver.subset(`<=${latestVersion}`, range)) { + return range + } + + // Extract lower bound from the range if possible + const lowerBound = extractLowerBound(range) + + if (lowerBound) { + return `>=${lowerBound.version} <=${latestVersion}` + } else { + return `<=${latestVersion}` + } +} + +// Extract the lower bound from a semver range +function extractLowerBound (range) { + const parsedRange = new semver.Range(range) + let lowerBound = null + + if (parsedRange.set && parsedRange.set.length > 0) { + for (const comparators of parsedRange.set) { + for (const comparator of comparators) { + if (comparator.operator === '>=' || comparator.operator === '>') { + if (!lowerBound || semver.gt(comparator.semver, lowerBound)) { + lowerBound = comparator.semver + } + } + } + } + } + + return lowerBound +} + async function assertPackage (name, version, dependencyVersionRange, external) { // Apply version cap from latests.json if available - // TODO: pinned versions? let cappedVersionRange = dependencyVersionRange + if (latests.latests[name]) { const latestVersion = latests.latests[name] - // If the range would allow versions beyond what we've tested, cap it - if (semver.validRange(dependencyVersionRange) && - !semver.subset(`<=${latestVersion}`, dependencyVersionRange)) { - cappedVersionRange = `${dependencyVersionRange} <=${latestVersion}` + + // Only process string version ranges + if (dependencyVersionRange && typeof dependencyVersionRange === 'string') { + cappedVersionRange = applyCap(dependencyVersionRange, latestVersion) } } - const dependencies = { [name]: cappedVersionRange } if (deps[name]) { await addDependencies(dependencies, name, cappedVersionRange) From 9ee322d106874d3f3a6569c15e944ce3c86d34f0 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Tue, 4 Mar 2025 15:39:49 -0500 Subject: [PATCH 03/15] Attempt to handle caret ranges --- scripts/install_plugin_modules.js | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index fbb918f7c1e..00d3d06bc69 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -108,6 +108,12 @@ function assertFolder (name, version) { // Helper function to apply version caps in a more readable way function applyCap (versionRange, latestVersion) { + // Handle caret ranges (e.g., "^3.0.7") + const caretRangeMatch = versionRange.match(/^\^(\d+\.\d+\.\d+)(.*)$/) + if (caretRangeMatch) { + return handleCaretRange(caretRangeMatch, latestVersion) + } + // Handle hyphen ranges (e.g., "24.8.0 - 24.9.0") const hyphenRangeMatch = versionRange.match(/^(\d+\.\d+\.\d+)\s*-\s*(\d+\.\d+\.\d+)(.*)$/) if (hyphenRangeMatch) { @@ -128,6 +134,43 @@ function applyCap (versionRange, latestVersion) { return versionRange } +// Handle caret ranges like "^3.0.7" +function handleCaretRange (match, latestVersion) { + const [, version, extraConstraints] = match + const parsed = semver.parse(version) + + // Calculate the upper bound implied by caret notation + let upperBound + if (parsed.major === 0) { + if (parsed.minor === 0) { + // ^0.0.x -> <0.0.(x+1) + upperBound = `0.0.${parsed.patch + 1}` + } else { + // ^0.y.x -> <0.(y+1).0 + upperBound = `0.${parsed.minor + 1}.0` + } + } else { + // ^x.y.z -> <(x+1).0.0 + upperBound = `${parsed.major + 1}.0.0` + } + + // Cap at the lower of: original caret upper bound or latest version + const effectiveLatest = semver.lt(upperBound, latestVersion) ? upperBound : latestVersion + + // Create properly formatted range that preserves caret semantics + let result = `>=${version} <${effectiveLatest}` + + // Add any extra constraints if they exist and would create a valid range + if (extraConstraints && extraConstraints.trim()) { + const combinedRange = `${result} ${extraConstraints.trim()}` + if (semver.validRange(combinedRange)) { + result = combinedRange + } + } + + return result +} + // Handle hyphen ranges like "24.8.0 - 24.9.0" function handleHyphenRange (match, latestVersion) { const [, lowerBound, upperBound, extraConstraints] = match From bb12c99fdad4ad0b862fef00b5bd9a4aa7c4b417 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Wed, 5 Mar 2025 08:38:24 -0500 Subject: [PATCH 04/15] Update latests --- .../src/helpers/latests.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json index 9a6c82b4f8a..7fcf085abb6 100644 --- a/packages/datadog-instrumentations/src/helpers/latests.json +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -3,7 +3,7 @@ "ENTER_PACKAGE_NAME_HERE" ], "latests": { - "aerospike": "6.0.2", + "aerospike": "6.1.0", "amqp10": "3.6.0", "amqplib": "0.10.5", "apollo-server-core": "3.13.0", @@ -35,7 +35,7 @@ "fs": "0.0.1-security", "generic-pool": "3.9.0", "@google-cloud/pubsub": "4.10.0", - "@graphql-tools/executor": "1.4.2", + "@graphql-tools/executor": "1.4.3", "graphql": "16.10.0", "@grpc/grpc-js": "1.12.6", "handlebars": "4.7.8", @@ -57,7 +57,7 @@ "koa": "2.16.0", "@koa/router": "13.1.0", "koa-router": "13.0.1", - "@langchain/core": "0.3.41", + "@langchain/core": "0.3.42", "@langchain/openai": "0.4.4", "ldapjs": "3.0.7", "limitd-client": "2.14.1", @@ -67,13 +67,13 @@ "microgateway-core": "3.3.5", "moleculer": "0.14.35", "mongodb-core": "3.2.7", - "mongodb": "6.14.0", - "mongoose": "8.11.0", + "mongodb": "6.14.2", + "mongoose": "8.12.1", "mquery": "5.0.0", "multer": "1.4.5-lts.1", "mysql": "2.18.1", "mysql2": "3.12.0", - "next": "15.2.0", + "next": "15.2.1", "node-serialize": "0.0.4", "nyc": "17.1.0", "openai": "4.86.1", @@ -100,8 +100,8 @@ "rhea": "3.0.3", "router": "1.3.8", "selenium-webdriver": "4.29.0", - "sequelize": "6.37.5", - "sharedb": "5.1.1", + "sequelize": "6.37.6", + "sharedb": "5.2.0", "tedious": "18.6.1", "undici": "7.4.0", "vitest": "3.0.7", From a7aff38caacd2c80874506da3185133b2033cd2e Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Wed, 5 Mar 2025 08:57:17 -0500 Subject: [PATCH 05/15] Fix eslint issues in outdated --- scripts/outdated.js | 228 ++++++++++++++++++++++---------------------- 1 file changed, 114 insertions(+), 114 deletions(-) diff --git a/scripts/outdated.js b/scripts/outdated.js index 48c3a2c2755..aaf9e40e6ad 100644 --- a/scripts/outdated.js +++ b/scripts/outdated.js @@ -1,129 +1,129 @@ /* eslint-disable no-console */ const { - getInternals, - npmView - } = require('./helpers/versioning') - const path = require('path') - const fs = require('fs') - - const latestsPath = path.join( - __dirname, - '..', - 'packages', - 'datadog-instrumentations', - 'src', - 'helpers', - 'latests.json' - ) - - // Get internal package names from existing getInternals helper - const internalsNames = Array.from(new Set(getInternals().map(n => n.name))) - .filter(x => typeof x === 'string' && x !== 'child_process' && !x.startsWith('node:')) - - // Initial structure with placeholder for pinned packages - const initialStructure = { - pinned: ['ENTER_PACKAGE_NAME_HERE'], - latests: {} - } - - /** + getInternals, + npmView +} = require('./helpers/versioning') +const path = require('path') +const fs = require('fs') + +const latestsPath = path.join( + __dirname, + '..', + 'packages', + 'datadog-instrumentations', + 'src', + 'helpers', + 'latests.json' +) + +// Get internal package names from existing getInternals helper +const internalsNames = Array.from(new Set(getInternals().map(n => n.name))) + .filter(x => typeof x === 'string' && x !== 'child_process' && !x.startsWith('node:')) + +// Initial structure with placeholder for pinned packages +const initialStructure = { + pinned: ['ENTER_PACKAGE_NAME_HERE'], + latests: {} +} + +/** * Updates latests.json with the current latest versions from npm */ - async function fix () { - console.log('Starting fix operation...') - console.log(`Found ${internalsNames.length} packages to process`) - - let outputData = initialStructure - if (fs.existsSync(latestsPath)) { - console.log('Found existing latests.json, loading it...') - outputData = require(latestsPath) - } - - const latests = {} - let processed = 0 - const total = internalsNames.length - - for (const name of internalsNames) { - processed++ - process.stdout.write(`Processing package ${processed}/${total}: ${name}...`) - - try { - const distTags = await npmView(name + ' dist-tags') - const latest = distTags.latest - if (latest) { - latests[name] = latest - process.stdout.write(` found version ${latest}\n`) - } else { - process.stdout.write(' WARNING: no version found\n') - console.log(`Warning: Could not fetch latest version for "${name}"`) - } - } catch (error) { - process.stdout.write(' ERROR\n') - console.error(`Error fetching version for "${name}":`, error.message) +async function fix () { + console.log('Starting fix operation...') + console.log(`Found ${internalsNames.length} packages to process`) + + let outputData = initialStructure + if (fs.existsSync(latestsPath)) { + console.log('Found existing latests.json, loading it...') + outputData = require(latestsPath) + } + + const latests = {} + let processed = 0 + const total = internalsNames.length + + for (const name of internalsNames) { + processed++ + process.stdout.write(`Processing package ${processed}/${total}: ${name}...`) + + try { + const distTags = await npmView(name + ' dist-tags') + const latest = distTags.latest + if (latest) { + latests[name] = latest + process.stdout.write(` found version ${latest}\n`) + } else { + process.stdout.write(' WARNING: no version found\n') + console.log(`Warning: Could not fetch latest version for "${name}"`) } + } catch (error) { + process.stdout.write(' ERROR\n') + console.error(`Error fetching version for "${name}":`, error.message) } - - outputData.latests = latests - console.log('\nWriting updated versions to latests.json...') - fs.writeFileSync(latestsPath, JSON.stringify(outputData, null, 2)) - console.log('Successfully updated latests.json') - console.log(`Processed ${total} packages`) } - - /** + + outputData.latests = latests + console.log('\nWriting updated versions to latests.json...') + fs.writeFileSync(latestsPath, JSON.stringify(outputData, null, 2)) + console.log('Successfully updated latests.json') + console.log(`Processed ${total} packages`) +} + +/** * Checks if latests.json matches current npm versions */ - async function check () { - console.log('Starting version check...') - - if (!fs.existsSync(latestsPath)) { - console.log('latests.json does not exist. Run with "fix" to create it.') +async function check () { + console.log('Starting version check...') + + if (!fs.existsSync(latestsPath)) { + console.log('latests.json does not exist. Run with "fix" to create it.') + process.exitCode = 1 + return + } + + const currentData = require(latestsPath) + console.log(`Found ${internalsNames.length} packages to check`) + + let processed = 0 + let mismatches = 0 + const total = internalsNames.length + + for (const name of internalsNames) { + processed++ + process.stdout.write(`Checking package ${processed}/${total}: ${name}...`) + + const latest = currentData.latests[name] + if (!latest) { + process.stdout.write(' MISSING\n') + console.log(`No latest version found for "${name}"`) process.exitCode = 1 - return + continue } - - const currentData = require(latestsPath) - console.log(`Found ${internalsNames.length} packages to check`) - - let processed = 0 - let mismatches = 0 - const total = internalsNames.length - - for (const name of internalsNames) { - processed++ - process.stdout.write(`Checking package ${processed}/${total}: ${name}...`) - - const latest = currentData.latests[name] - if (!latest) { - process.stdout.write(' MISSING\n') - console.log(`No latest version found for "${name}"`) + + try { + const distTags = await npmView(name + ' dist-tags') + const npmLatest = distTags.latest + if (npmLatest !== latest) { + process.stdout.write(' MISMATCH\n') + console.log(`"latests.json: is not up to date for "${name}": expected "${npmLatest}", got "${latest}"`) process.exitCode = 1 - continue + mismatches++ + } else { + process.stdout.write(' OK\n') } - - try { - const distTags = await npmView(name + ' dist-tags') - const npmLatest = distTags.latest - if (npmLatest !== latest) { - process.stdout.write(' MISMATCH\n') - console.log(`"latests.json: is not up to date for "${name}": expected "${npmLatest}", got "${latest}"`) - process.exitCode = 1 - mismatches++ - } else { - process.stdout.write(' OK\n') - } - } catch (error) { - process.stdout.write(' ERROR\n') - console.error(`Error checking version for "${name}":`, error.message) - } - } - - console.log('\nCheck completed:') - console.log(`- Total packages checked: ${total}`) - console.log(`- Version mismatches found: ${mismatches}`) - if (mismatches > 0) { - console.log('Run with "fix" to update versions') + } catch (error) { + process.stdout.write(' ERROR\n') + console.error(`Error checking version for "${name}":`, error.message) } } - if (process.argv.includes('fix')) fix() - else check() \ No newline at end of file + + console.log('\nCheck completed:') + console.log(`- Total packages checked: ${total}`) + console.log(`- Version mismatches found: ${mismatches}`) + if (mismatches > 0) { + console.log('Run with "fix" to update versions') + } +} +if (process.argv.includes('fix')) fix() +else check() From 01bb8df74f632faa5b5955c3320d9620bd8a3398 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Wed, 5 Mar 2025 09:21:15 -0500 Subject: [PATCH 06/15] Attempt to handle stable non-latest versions --- .../src/helpers/latests.json | 14 ++-- scripts/outdated.js | 76 +++++++++++++++---- 2 files changed, 69 insertions(+), 21 deletions(-) diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json index 7fcf085abb6..7da2057c78f 100644 --- a/packages/datadog-instrumentations/src/helpers/latests.json +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -15,8 +15,8 @@ "aws-sdk": "2.1692.0", "@azure/functions": "4.6.1", "bluebird": "3.7.2", - "body-parser": "1.20.3", - "bunyan": "1.8.15", + "body-parser": "2.1.0", + "bunyan": "2.0.5", "cassandra-driver": "4.8.0", "connect": "3.7.0", "cookie-parser": "1.4.7", @@ -29,10 +29,10 @@ "elasticsearch": "16.7.3", "express-mongo-sanitize": "2.2.0", "express-session": "1.18.1", - "express": "4.21.2", + "express": "5.0.1", "fastify": "5.2.1", "find-my-way": "9.2.0", - "fs": "0.0.1-security", + "fs": "0.0.2", "generic-pool": "3.9.0", "@google-cloud/pubsub": "4.10.0", "@graphql-tools/executor": "1.4.3", @@ -92,17 +92,17 @@ "promise": "8.3.0", "protobufjs": "7.4.0", "pug": "3.0.3", - "q": "1.5.1", + "q": "2.0.3", "@node-redis/client": "1.0.6", "@redis/client": "1.6.0", "redis": "4.7.0", "restify": "11.1.0", "rhea": "3.0.3", - "router": "1.3.8", + "router": "2.1.0", "selenium-webdriver": "4.29.0", "sequelize": "6.37.6", "sharedb": "5.2.0", - "tedious": "18.6.1", + "tedious": "19.0.0", "undici": "7.4.0", "vitest": "3.0.7", "@vitest/runner": "3.0.7", diff --git a/scripts/outdated.js b/scripts/outdated.js index aaf9e40e6ad..1da78764940 100644 --- a/scripts/outdated.js +++ b/scripts/outdated.js @@ -5,6 +5,7 @@ const { } = require('./helpers/versioning') const path = require('path') const fs = require('fs') +const semver = require('semver') const latestsPath = path.join( __dirname, @@ -20,15 +21,58 @@ const latestsPath = path.join( const internalsNames = Array.from(new Set(getInternals().map(n => n.name))) .filter(x => typeof x === 'string' && x !== 'child_process' && !x.startsWith('node:')) -// Initial structure with placeholder for pinned packages +// Initial structure with placeholder for configuration const initialStructure = { pinned: ['ENTER_PACKAGE_NAME_HERE'], + onlyUseLatestTag: ['ENTER_PACKAGE_NAME_HERE'], latests: {} } /** - * Updates latests.json with the current latest versions from npm - */ + * Gets the highest version that's compatible with our instrumentation + * This handles cases where a package has newer versions that might break compatibility + */ +async function getHighestCompatibleVersion (name, config = {}) { + try { + // Get all distribution tags (including 'latest') + const distTags = await npmView(name + ' dist-tags') + + // Get the latest tagged version + const latestTagged = distTags.latest + + if (!latestTagged) { + console.log(`Warning: Could not fetch latest version for "${name}"`) + return null + } + + // If package is in the onlyUseLatestTag list, always use the 'latest' tag + if (config.onlyUseLatestTag && config.onlyUseLatestTag.includes(name)) { + return latestTagged + } + + // Get all available versions + const allVersions = await npmView(name + ' versions') + + // Find the highest non-prerelease version available + const stableVersions = allVersions.filter(v => !semver.prerelease(v)) + const highestStableVersion = stableVersions.sort(semver.compare).pop() + + // Use the highest stable version if it's greater than the latest tag + if (highestStableVersion && semver.gt(highestStableVersion, latestTagged)) { + process.stdout.write(` found version ${highestStableVersion} (higher than 'latest' tag ${latestTagged})`) + return highestStableVersion + } + + return latestTagged + } catch (error) { + console.error(`Error fetching version for "${name}":`, error.message) + return null + } +} + +/** + * Updates latests.json with the current latest versions from npm + */ async function fix () { console.log('Starting fix operation...') console.log(`Found ${internalsNames.length} packages to process`) @@ -48,11 +92,10 @@ async function fix () { process.stdout.write(`Processing package ${processed}/${total}: ${name}...`) try { - const distTags = await npmView(name + ' dist-tags') - const latest = distTags.latest - if (latest) { - latests[name] = latest - process.stdout.write(` found version ${latest}\n`) + const latestVersion = await getHighestCompatibleVersion(name, outputData) + if (latestVersion) { + latests[name] = latestVersion + process.stdout.write(` found version ${latestVersion}\n`) } else { process.stdout.write(' WARNING: no version found\n') console.log(`Warning: Could not fetch latest version for "${name}"`) @@ -71,8 +114,8 @@ async function fix () { } /** - * Checks if latests.json matches current npm versions - */ + * Checks if latests.json matches current npm versions + */ async function check () { console.log('Starting version check...') @@ -102,11 +145,16 @@ async function check () { } try { - const distTags = await npmView(name + ' dist-tags') - const npmLatest = distTags.latest - if (npmLatest !== latest) { + const latestVersion = await getHighestCompatibleVersion(name, currentData) + if (!latestVersion) { + process.stdout.write(' ERROR\n') + console.error(`Error fetching latest version for "${name}"`) + continue + } + + if (latestVersion !== latest) { process.stdout.write(' MISMATCH\n') - console.log(`"latests.json: is not up to date for "${name}": expected "${npmLatest}", got "${latest}"`) + console.log(`"latests.json: is not up to date for "${name}": expected "${latestVersion}", got "${latest}"`) process.exitCode = 1 mismatches++ } else { From 67809b21525ea534d1fd4404eb0b8073bc1e5547 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 7 Mar 2025 16:00:30 -0500 Subject: [PATCH 07/15] Cleanup outdated to handle various one-off issues --- .../src/helpers/latests.json | 37 ++-- scripts/outdated.js | 192 ++++++++++-------- 2 files changed, 123 insertions(+), 106 deletions(-) diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json index 7da2057c78f..2a623b36825 100644 --- a/packages/datadog-instrumentations/src/helpers/latests.json +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -1,9 +1,15 @@ { "pinned": [ - "ENTER_PACKAGE_NAME_HERE" + "fastify" + ], + "onlyUseLatestTag": [], + "ignored": [ + "aerospike", + "mariadb", + "microgateway-core", + "winston" ], "latests": { - "aerospike": "6.1.0", "amqp10": "3.6.0", "amqplib": "0.10.5", "apollo-server-core": "3.13.0", @@ -13,7 +19,7 @@ "@smithy/smithy-client": "4.1.6", "@aws-sdk/smithy-client": "3.374.0", "aws-sdk": "2.1692.0", - "@azure/functions": "4.6.1", + "@azure/functions": "4.7.0", "bluebird": "3.7.2", "body-parser": "2.1.0", "bunyan": "2.0.5", @@ -30,18 +36,18 @@ "express-mongo-sanitize": "2.2.0", "express-session": "1.18.1", "express": "5.0.1", - "fastify": "5.2.1", + "fastify": "4.28.1", "find-my-way": "9.2.0", "fs": "0.0.2", "generic-pool": "3.9.0", "@google-cloud/pubsub": "4.10.0", - "@graphql-tools/executor": "1.4.3", + "@graphql-tools/executor": "1.4.4", "graphql": "16.10.0", "@grpc/grpc-js": "1.12.6", "handlebars": "4.7.8", - "@hapi/hapi": "21.3.12", + "@hapi/hapi": "21.4.0", "hapi": "18.1.0", - "ioredis": "5.5.0", + "ioredis": "5.6.0", "jest-environment-node": "29.7.0", "jest-environment-jsdom": "29.7.0", "@jest/core": "29.7.0", @@ -62,9 +68,7 @@ "ldapjs": "3.0.7", "limitd-client": "2.14.1", "lodash": "4.17.21", - "mariadb": "3.4.0", "memcached": "2.2.2", - "microgateway-core": "3.3.5", "moleculer": "0.14.35", "mongodb-core": "3.2.7", "mongodb": "6.14.2", @@ -72,11 +76,11 @@ "mquery": "5.0.0", "multer": "1.4.5-lts.1", "mysql": "2.18.1", - "mysql2": "3.12.0", + "mysql2": "3.13.0", "next": "15.2.1", "node-serialize": "0.0.4", "nyc": "17.1.0", - "openai": "4.86.1", + "openai": "4.86.2", "@opensearch-project/opensearch": "3.4.0", "oracledb": "6.8.0", "paperplane": "3.1.2", @@ -86,8 +90,8 @@ "pg": "8.13.3", "pino": "9.6.0", "pino-pretty": "13.0.0", - "@playwright/test": "1.50.1", - "playwright": "1.50.1", + "@playwright/test": "1.51.0", + "playwright": "1.51.0", "promise-js": "0.0.7", "promise": "8.3.0", "protobufjs": "7.4.0", @@ -104,9 +108,8 @@ "sharedb": "5.2.0", "tedious": "19.0.0", "undici": "7.4.0", - "vitest": "3.0.7", - "@vitest/runner": "3.0.7", - "when": "3.7.8", - "winston": "3.17.0" + "vitest": "3.0.8", + "@vitest/runner": "3.0.8", + "when": "3.7.8" } } \ No newline at end of file diff --git a/scripts/outdated.js b/scripts/outdated.js index 1da78764940..25d132a024c 100644 --- a/scripts/outdated.js +++ b/scripts/outdated.js @@ -1,11 +1,11 @@ /* eslint-disable no-console */ -const { - getInternals, - npmView -} = require('./helpers/versioning') +'use strict' + +const { getInternals } = require('./helpers/versioning') const path = require('path') const fs = require('fs') const semver = require('semver') +const childProcess = require('child_process') const latestsPath = path.join( __dirname, @@ -17,25 +17,83 @@ const latestsPath = path.join( 'latests.json' ) -// Get internal package names from existing getInternals helper const internalsNames = Array.from(new Set(getInternals().map(n => n.name))) .filter(x => typeof x === 'string' && x !== 'child_process' && !x.startsWith('node:')) -// Initial structure with placeholder for configuration -const initialStructure = { - pinned: ['ENTER_PACKAGE_NAME_HERE'], - onlyUseLatestTag: ['ENTER_PACKAGE_NAME_HERE'], +// Packages that should be ignored during version checking - these won't be included in latests.json +const IGNORED_PACKAGES = [ + // Add package names here + 'aerospike', // I think this is due to architecture issues? + 'mariadb', // mariadb esm tests were failing + 'microgateway-core', // 'microgateway-core' was failing to find a directory + 'winston' // winston esm tests were failing +] + +// Packages that should be pinned to specific versions +const PINNED_PACKAGES = { + // Example: 'express': '4.17.3' + fastify: '4.28.1' // v5+ is not supported +} + +// Packages that should only use the 'latest' tag (not 'next' or other dist-tags) +// Some packages have a next tag that is a stable semver version +const ONLY_USE_LATEST_TAG = [ + // Example: 'router' +] + +// Initial structure for latests.json that will be recreated each run +const outputData = { + pinned: Object.keys(PINNED_PACKAGES), + onlyUseLatestTag: ONLY_USE_LATEST_TAG, + ignored: IGNORED_PACKAGES, latests: {} } -/** - * Gets the highest version that's compatible with our instrumentation - * This handles cases where a package has newer versions that might break compatibility - */ -async function getHighestCompatibleVersion (name, config = {}) { +function npmView (input) { + return new Promise((resolve, reject) => { + childProcess.exec(`npm view ${input} --json`, (err, stdout) => { + if (err) { + reject(err) + return + } + try { + resolve(JSON.parse(stdout.toString('utf8'))) + } catch (e) { + reject(new Error(`Failed to parse npm output for ${input}: ${e.message}`)) + } + }) + }) +} + +async function getHighestCompatibleVersion (name) { try { + if (IGNORED_PACKAGES.includes(name)) { + console.log(`Skipping "${name}" as it's in the ignored list`) + return null + } + + // If package is hardcoded as pinned, return the pinned version but also check latest + // this is for logging purposes + if (PINNED_PACKAGES[name]) { + const pinnedVersion = PINNED_PACKAGES[name] + + try { + const distTags = await npmView(`${name} dist-tags`) + const latestTagged = distTags.latest + + if (latestTagged && semver.gt(latestTagged, pinnedVersion)) { + console.log(`Note: "${name}" is pinned to ${pinnedVersion}, but ${latestTagged} is available`) + } + } catch (err) { + // Just log the error but continue with the pinned version + console.log(`Warning: Could not fetch latest version for pinned package "${name}": ${err.message}`) + } + + return pinnedVersion + } + // Get all distribution tags (including 'latest') - const distTags = await npmView(name + ' dist-tags') + const distTags = await npmView(`${name} dist-tags`) // Get the latest tagged version const latestTagged = distTags.latest @@ -46,12 +104,12 @@ async function getHighestCompatibleVersion (name, config = {}) { } // If package is in the onlyUseLatestTag list, always use the 'latest' tag - if (config.onlyUseLatestTag && config.onlyUseLatestTag.includes(name)) { + if (ONLY_USE_LATEST_TAG.includes(name)) { return latestTagged } // Get all available versions - const allVersions = await npmView(name + ' versions') + const allVersions = await npmView(`${name} versions`) // Find the highest non-prerelease version available const stableVersions = allVersions.filter(v => !semver.prerelease(v)) @@ -77,12 +135,7 @@ async function fix () { console.log('Starting fix operation...') console.log(`Found ${internalsNames.length} packages to process`) - let outputData = initialStructure - if (fs.existsSync(latestsPath)) { - console.log('Found existing latests.json, loading it...') - outputData = require(latestsPath) - } - + // Process packages const latests = {} let processed = 0 const total = internalsNames.length @@ -91,87 +144,48 @@ async function fix () { processed++ process.stdout.write(`Processing package ${processed}/${total}: ${name}...`) + // Skip ignored packages + if (IGNORED_PACKAGES.includes(name)) { + process.stdout.write(' IGNORED\n') + continue + } + try { - const latestVersion = await getHighestCompatibleVersion(name, outputData) + // Handle hardcoded pinned packages + if (PINNED_PACKAGES[name]) { + const pinnedVersion = PINNED_PACKAGES[name] + latests[name] = pinnedVersion + process.stdout.write(` PINNED to version ${pinnedVersion}\n`) + continue + } + + // Normal package processing + const latestVersion = await getHighestCompatibleVersion(name) if (latestVersion) { latests[name] = latestVersion process.stdout.write(` found version ${latestVersion}\n`) } else { process.stdout.write(' WARNING: no version found\n') - console.log(`Warning: Could not fetch latest version for "${name}"`) } } catch (error) { process.stdout.write(' ERROR\n') - console.error(`Error fetching version for "${name}":`, error.message) + console.error(`Error processing "${name}":`, error.message) } } + // Update the output data outputData.latests = latests - console.log('\nWriting updated versions to latests.json...') - fs.writeFileSync(latestsPath, JSON.stringify(outputData, null, 2)) - console.log('Successfully updated latests.json') - console.log(`Processed ${total} packages`) -} -/** - * Checks if latests.json matches current npm versions - */ -async function check () { - console.log('Starting version check...') - - if (!fs.existsSync(latestsPath)) { - console.log('latests.json does not exist. Run with "fix" to create it.') - process.exitCode = 1 - return - } - - const currentData = require(latestsPath) - console.log(`Found ${internalsNames.length} packages to check`) - - let processed = 0 - let mismatches = 0 - const total = internalsNames.length - - for (const name of internalsNames) { - processed++ - process.stdout.write(`Checking package ${processed}/${total}: ${name}...`) - - const latest = currentData.latests[name] - if (!latest) { - process.stdout.write(' MISSING\n') - console.log(`No latest version found for "${name}"`) - process.exitCode = 1 - continue - } + // Write the updated configuration with a comment at the top + console.log('\nWriting updated versions to latests.json...') - try { - const latestVersion = await getHighestCompatibleVersion(name, currentData) - if (!latestVersion) { - process.stdout.write(' ERROR\n') - console.error(`Error fetching latest version for "${name}"`) - continue - } + // Convert to JSON with proper indentation + const jsonContent = JSON.stringify(outputData, null, 2) - if (latestVersion !== latest) { - process.stdout.write(' MISMATCH\n') - console.log(`"latests.json: is not up to date for "${name}": expected "${latestVersion}", got "${latest}"`) - process.exitCode = 1 - mismatches++ - } else { - process.stdout.write(' OK\n') - } - } catch (error) { - process.stdout.write(' ERROR\n') - console.error(`Error checking version for "${name}":`, error.message) - } - } + fs.writeFileSync(latestsPath, jsonContent) - console.log('\nCheck completed:') - console.log(`- Total packages checked: ${total}`) - console.log(`- Version mismatches found: ${mismatches}`) - if (mismatches > 0) { - console.log('Run with "fix" to update versions') - } + console.log('Successfully updated latests.json') + console.log(`Processed ${total} packages`) } -if (process.argv.includes('fix')) fix() -else check() + +fix() From db3eb947a881391c7b8379e18c8d61f508dcba73 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 7 Mar 2025 17:14:48 -0500 Subject: [PATCH 08/15] Update comment and add newline --- scripts/helpers/versioning.js | 2 +- scripts/outdated.js | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/helpers/versioning.js b/scripts/helpers/versioning.js index 69cdec41df7..b51eaafee8b 100644 --- a/scripts/helpers/versioning.js +++ b/scripts/helpers/versioning.js @@ -75,4 +75,4 @@ module.exports = { npmView, loadInstFile, getInternals -} \ No newline at end of file +} diff --git a/scripts/outdated.js b/scripts/outdated.js index 25d132a024c..50302b12e5e 100644 --- a/scripts/outdated.js +++ b/scripts/outdated.js @@ -92,7 +92,7 @@ async function getHighestCompatibleVersion (name) { return pinnedVersion } - // Get all distribution tags (including 'latest') + // ideally we can just use `latest` tag, but a few use `next` const distTags = await npmView(`${name} dist-tags`) // Get the latest tagged version @@ -128,14 +128,10 @@ async function getHighestCompatibleVersion (name) { } } -/** - * Updates latests.json with the current latest versions from npm - */ async function fix () { console.log('Starting fix operation...') console.log(`Found ${internalsNames.length} packages to process`) - // Process packages const latests = {} let processed = 0 const total = internalsNames.length From 3f1dda541c156254aba086731de2f777a906e1ab Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Thu, 20 Mar 2025 12:42:30 -0400 Subject: [PATCH 09/15] Attempt to limit capping to within major versions --- scripts/install_plugin_modules.js | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index 00d3d06bc69..53597cbe36a 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -246,18 +246,34 @@ async function assertPackage (name, version, dependencyVersionRange, external) { // Apply version cap from latests.json if available let cappedVersionRange = dependencyVersionRange - if (latests.latests[name]) { - const latestVersion = latests.latests[name] - - // Only process string version ranges - if (dependencyVersionRange && typeof dependencyVersionRange === 'string') { - cappedVersionRange = applyCap(dependencyVersionRange, latestVersion) + // Handle simple major version numbers like "1" or exact versions like "1.0.0" + if (dependencyVersionRange) { + const simpleVersionMatch = dependencyVersionRange.match(/^(\d+)$/) + const exactVersionMatch = dependencyVersionRange.match(/^(\d+)\.(\d+)\.(\d+)$/) + + // Attempt to fix issue seen with fastify + if (simpleVersionMatch) { + // For simple major versions like "1", restrict to that major version only + const majorVersion = simpleVersionMatch[1] + cappedVersionRange = `>=${majorVersion}.0.0 <${parseInt(majorVersion) + 1}.0.0` + } else if (exactVersionMatch) { + // For exact versions like "1.0.0", still restrict to the same major version + const majorVersion = exactVersionMatch[1] + cappedVersionRange = `>=${dependencyVersionRange} <${parseInt(majorVersion) + 1}.0.0` + } else if (latests.latests[name]) { + // Apply normal capping for other version ranges + const latestVersion = latests.latests[name] + if (typeof dependencyVersionRange === 'string') { + cappedVersionRange = applyCap(dependencyVersionRange, latestVersion) + } } } + const dependencies = { [name]: cappedVersionRange } if (deps[name]) { await addDependencies(dependencies, name, cappedVersionRange) } + const pkg = { name: [name, sha1(name).substr(0, 8), sha1(version)].filter(val => val).join('-'), version: '1.0.0', @@ -279,7 +295,6 @@ async function assertPackage (name, version, dependencyVersionRange, external) { } fs.writeFileSync(filename(name, version, 'package.json'), JSON.stringify(pkg, null, 2) + '\n') } - async function addDependencies (dependencies, name, versionRange) { const versionList = await getVersionList(name) const version = semver.maxSatisfying(versionList, versionRange) From d0def31f3e3e9ebf63ad732c97cee185c062ffc9 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Thu, 20 Mar 2025 12:59:06 -0400 Subject: [PATCH 10/15] Use exact version when specified --- scripts/install_plugin_modules.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index 53597cbe36a..2c42ef872d0 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -257,9 +257,9 @@ async function assertPackage (name, version, dependencyVersionRange, external) { const majorVersion = simpleVersionMatch[1] cappedVersionRange = `>=${majorVersion}.0.0 <${parseInt(majorVersion) + 1}.0.0` } else if (exactVersionMatch) { - // For exact versions like "1.0.0", still restrict to the same major version - const majorVersion = exactVersionMatch[1] - cappedVersionRange = `>=${dependencyVersionRange} <${parseInt(majorVersion) + 1}.0.0` + // For exact versions like "1.0.0" or "2.2.2", use exactly that version + // Don't allow newer versions even within the same major + cappedVersionRange = dependencyVersionRange } else if (latests.latests[name]) { // Apply normal capping for other version ranges const latestVersion = latests.latests[name] From ea484e19512a4286e393dcd96a4cb97ec222f6ea Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 4 Apr 2025 08:36:29 -0400 Subject: [PATCH 11/15] Add workflow file to create PR --- .../auto-bump-test-package-versions.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/auto-bump-test-package-versions.yml diff --git a/.github/workflows/auto-bump-test-package-versions.yml b/.github/workflows/auto-bump-test-package-versions.yml new file mode 100644 index 00000000000..b7d5dfbf5cb --- /dev/null +++ b/.github/workflows/auto-bump-test-package-versions.yml @@ -0,0 +1,53 @@ +name: Auto bump latest tested NPM packages + +on: + schedule: + - cron: '0 0 * * 0' # Every Sunday at midnight + workflow_dispatch: + +jobs: + bump_package_versions: + runs-on: ubuntu-latest + permissions: + actions: read # read secrets + contents: write # Creates a branch + pull-requests: write # Creates a PR + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Setup Node.js + uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # v4.3.0 + with: + node-version: '18' + + - name: Install dependencies + run: yarn install + + - name: Update package versions in latests.json + run: node scripts/outdated.js + + - name: Create Pull Request + id: pr + uses: peter-evans/create-pull-request@dd2324fc52d5d43c699a5636bcf19fceaa70c284 # v7.0.7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: "bot/test-package-versions-bump" + commit-message: "[Test Package Versions Bump]" + delete-branch: true + base: master + title: "[Test Package Versions Bump] Updating package versions" + labels: dependencies + body: | + Automated update of the latest versions of NPM packages used in tests. + + # If desired we can enable automerge with the necessary PAT created + # - name: Enable Pull Request Automerge + # if: steps.pr.outputs.pull-request-operation == 'created' + # uses: peter-evans/enable-pull-request-automerge@a660677d5469627102a1c1e11409dd063606628d # v3.0.0 + # with: + # token: ${{ secrets.GHA_PAT }} + # pull-request-number: ${{ steps.pr.outputs.pull-request-number }} From 88c20da6949ab9366fd6802eef2a02c9ba901622 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 4 Apr 2025 08:48:39 -0400 Subject: [PATCH 12/15] Update latests.json --- .../src/helpers/latests.json | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json index 2a623b36825..55e79924966 100644 --- a/packages/datadog-instrumentations/src/helpers/latests.json +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -11,17 +11,17 @@ ], "latests": { "amqp10": "3.6.0", - "amqplib": "0.10.5", + "amqplib": "0.10.7", "apollo-server-core": "3.13.0", "@apollo/server": "4.11.3", "@apollo/gateway": "2.10.0", "avsc": "5.7.7", - "@smithy/smithy-client": "4.1.6", + "@smithy/smithy-client": "4.2.0", "@aws-sdk/smithy-client": "3.374.0", "aws-sdk": "2.1692.0", "@azure/functions": "4.7.0", "bluebird": "3.7.2", - "body-parser": "2.1.0", + "body-parser": "2.2.0", "bunyan": "2.0.5", "cassandra-driver": "4.8.0", "connect": "3.7.0", @@ -29,21 +29,21 @@ "cookie": "1.0.2", "couchbase": "4.4.5", "@cucumber/cucumber": "11.2.0", - "cypress": "14.1.0", - "@elastic/transport": "8.9.4", + "cypress": "14.2.1", + "@elastic/transport": "8.9.5", "@elastic/elasticsearch": "8.17.1", "elasticsearch": "16.7.3", "express-mongo-sanitize": "2.2.0", "express-session": "1.18.1", - "express": "5.0.1", + "express": "5.1.0", "fastify": "4.28.1", - "find-my-way": "9.2.0", + "find-my-way": "9.3.0", "fs": "0.0.2", "generic-pool": "3.9.0", - "@google-cloud/pubsub": "4.10.0", - "@graphql-tools/executor": "1.4.4", + "@google-cloud/pubsub": "4.11.0", + "@graphql-tools/executor": "1.4.7", "graphql": "16.10.0", - "@grpc/grpc-js": "1.12.6", + "@grpc/grpc-js": "1.13.2", "handlebars": "4.7.8", "@hapi/hapi": "21.4.0", "hapi": "18.1.0", @@ -63,35 +63,35 @@ "koa": "2.16.0", "@koa/router": "13.1.0", "koa-router": "13.0.1", - "@langchain/core": "0.3.42", - "@langchain/openai": "0.4.4", + "@langchain/core": "0.3.43", + "@langchain/openai": "0.5.2", "ldapjs": "3.0.7", "limitd-client": "2.14.1", "lodash": "4.17.21", "memcached": "2.2.2", "moleculer": "0.14.35", "mongodb-core": "3.2.7", - "mongodb": "6.14.2", - "mongoose": "8.12.1", + "mongodb": "6.15.0", + "mongoose": "8.13.2", "mquery": "5.0.0", - "multer": "1.4.5-lts.1", + "multer": "1.4.5-lts.2", "mysql": "2.18.1", - "mysql2": "3.13.0", - "next": "15.2.1", + "mysql2": "3.14.0", + "next": "15.2.4", "node-serialize": "0.0.4", "nyc": "17.1.0", - "openai": "4.86.2", - "@opensearch-project/opensearch": "3.4.0", + "openai": "4.91.1", + "@opensearch-project/opensearch": "3.5.0", "oracledb": "6.8.0", "paperplane": "3.1.2", "passport-http": "0.3.0", "passport-local": "1.0.0", "passport": "0.7.0", - "pg": "8.13.3", + "pg": "8.14.1", "pino": "9.6.0", "pino-pretty": "13.0.0", - "@playwright/test": "1.51.0", - "playwright": "1.51.0", + "@playwright/test": "1.51.1", + "playwright": "1.51.1", "promise-js": "0.0.7", "promise": "8.3.0", "protobufjs": "7.4.0", @@ -101,15 +101,15 @@ "@redis/client": "1.6.0", "redis": "4.7.0", "restify": "11.1.0", - "rhea": "3.0.3", - "router": "2.1.0", - "selenium-webdriver": "4.29.0", - "sequelize": "6.37.6", - "sharedb": "5.2.0", + "rhea": "3.0.4", + "router": "2.2.0", + "selenium-webdriver": "4.30.0", + "sequelize": "6.37.7", + "sharedb": "5.2.1", "tedious": "19.0.0", - "undici": "7.4.0", - "vitest": "3.0.8", - "@vitest/runner": "3.0.8", + "undici": "7.7.0", + "vitest": "3.1.1", + "@vitest/runner": "3.1.1", "when": "3.7.8" } } \ No newline at end of file From aa47ca3550fbc026825ea30d3ea3093a6c97a522 Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 4 Apr 2025 11:25:21 -0400 Subject: [PATCH 13/15] Pin express - unsure if necessary though --- packages/datadog-instrumentations/src/helpers/latests.json | 5 +++-- scripts/outdated.js | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json index 55e79924966..41e5c0a0dbf 100644 --- a/packages/datadog-instrumentations/src/helpers/latests.json +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -1,6 +1,7 @@ { "pinned": [ - "fastify" + "fastify", + "express" ], "onlyUseLatestTag": [], "ignored": [ @@ -35,7 +36,7 @@ "elasticsearch": "16.7.3", "express-mongo-sanitize": "2.2.0", "express-session": "1.18.1", - "express": "5.1.0", + "express": "4.21.2", "fastify": "4.28.1", "find-my-way": "9.3.0", "fs": "0.0.2", diff --git a/scripts/outdated.js b/scripts/outdated.js index 50302b12e5e..91789855108 100644 --- a/scripts/outdated.js +++ b/scripts/outdated.js @@ -32,7 +32,8 @@ const IGNORED_PACKAGES = [ // Packages that should be pinned to specific versions const PINNED_PACKAGES = { // Example: 'express': '4.17.3' - fastify: '4.28.1' // v5+ is not supported + fastify: '4.28.1', // v5+ is not supported + express: '4.21.2' } // Packages that should only use the 'latest' tag (not 'next' or other dist-tags) From 660a84be493db4d758140b25b84fb21c8ed9d8eb Mon Sep 17 00:00:00 2001 From: Steven Bouwkamp Date: Fri, 4 Apr 2025 13:36:37 -0400 Subject: [PATCH 14/15] Cap express to 5.0.1 --- packages/datadog-instrumentations/src/helpers/latests.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json index 41e5c0a0dbf..4ea49974e79 100644 --- a/packages/datadog-instrumentations/src/helpers/latests.json +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -36,7 +36,7 @@ "elasticsearch": "16.7.3", "express-mongo-sanitize": "2.2.0", "express-session": "1.18.1", - "express": "4.21.2", + "express": "5.0.1", "fastify": "4.28.1", "find-my-way": "9.3.0", "fs": "0.0.2", From 5318d3095ec67ca5766c0b855eb71b33c9f76284 Mon Sep 17 00:00:00 2001 From: Roch Devost Date: Fri, 20 Jun 2025 12:54:22 -0400 Subject: [PATCH 15/15] simplify capping code (#5604) --- .../src/helpers/latests.json | 2 +- scripts/install_plugin_modules.js | 165 +----------------- 2 files changed, 5 insertions(+), 162 deletions(-) diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json index 4ea49974e79..9898f34982c 100644 --- a/packages/datadog-instrumentations/src/helpers/latests.json +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -113,4 +113,4 @@ "@vitest/runner": "3.1.1", "when": "3.7.8" } -} \ No newline at end of file +} diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index 2c42ef872d0..929106e9c13 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -106,168 +106,11 @@ function assertFolder (name, version) { } } -// Helper function to apply version caps in a more readable way -function applyCap (versionRange, latestVersion) { - // Handle caret ranges (e.g., "^3.0.7") - const caretRangeMatch = versionRange.match(/^\^(\d+\.\d+\.\d+)(.*)$/) - if (caretRangeMatch) { - return handleCaretRange(caretRangeMatch, latestVersion) - } - - // Handle hyphen ranges (e.g., "24.8.0 - 24.9.0") - const hyphenRangeMatch = versionRange.match(/^(\d+\.\d+\.\d+)\s*-\s*(\d+\.\d+\.\d+)(.*)$/) - if (hyphenRangeMatch) { - return handleHyphenRange(hyphenRangeMatch, latestVersion) - } - - // Handle exact versions (e.g., "24.8.0") - if (semver.valid(versionRange)) { - return handleExactVersion(versionRange, latestVersion) - } - - // Handle other valid semver ranges - if (semver.validRange(versionRange)) { - return handleValidRange(versionRange, latestVersion) - } - - // If nothing else matched, return the original range - return versionRange -} - -// Handle caret ranges like "^3.0.7" -function handleCaretRange (match, latestVersion) { - const [, version, extraConstraints] = match - const parsed = semver.parse(version) - - // Calculate the upper bound implied by caret notation - let upperBound - if (parsed.major === 0) { - if (parsed.minor === 0) { - // ^0.0.x -> <0.0.(x+1) - upperBound = `0.0.${parsed.patch + 1}` - } else { - // ^0.y.x -> <0.(y+1).0 - upperBound = `0.${parsed.minor + 1}.0` - } - } else { - // ^x.y.z -> <(x+1).0.0 - upperBound = `${parsed.major + 1}.0.0` - } - - // Cap at the lower of: original caret upper bound or latest version - const effectiveLatest = semver.lt(upperBound, latestVersion) ? upperBound : latestVersion - - // Create properly formatted range that preserves caret semantics - let result = `>=${version} <${effectiveLatest}` - - // Add any extra constraints if they exist and would create a valid range - if (extraConstraints && extraConstraints.trim()) { - const combinedRange = `${result} ${extraConstraints.trim()}` - if (semver.validRange(combinedRange)) { - result = combinedRange - } - } - - return result -} - -// Handle hyphen ranges like "24.8.0 - 24.9.0" -function handleHyphenRange (match, latestVersion) { - const [, lowerBound, upperBound, extraConstraints] = match - - // Cap at the lower of: original upper bound or latest version - const effectiveUpper = semver.lt(upperBound, latestVersion) ? upperBound : latestVersion - - // Create properly formatted range - let result = `>=${lowerBound} <=${effectiveUpper}` - - // Add any extra constraints if they exist and would create a valid range - if (extraConstraints && extraConstraints.trim()) { - const combinedRange = `${result} ${extraConstraints.trim()}` - if (semver.validRange(combinedRange)) { - result = combinedRange - } - } - - return result -} - -// Handle exact versions like "24.8.0" -function handleExactVersion (version, latestVersion) { - const exactVersion = semver.clean(version) - - // If exact version is too high, cap it - if (semver.gt(exactVersion, latestVersion)) { - return latestVersion - } - - // Otherwise keep exact version with cap - return `${exactVersion} <=${latestVersion}` -} - -// Handle general semver ranges -function handleValidRange (range, latestVersion) { - // Only apply cap if necessary - if (semver.subset(`<=${latestVersion}`, range)) { - return range - } - - // Extract lower bound from the range if possible - const lowerBound = extractLowerBound(range) - - if (lowerBound) { - return `>=${lowerBound.version} <=${latestVersion}` - } else { - return `<=${latestVersion}` - } -} - -// Extract the lower bound from a semver range -function extractLowerBound (range) { - const parsedRange = new semver.Range(range) - let lowerBound = null - - if (parsedRange.set && parsedRange.set.length > 0) { - for (const comparators of parsedRange.set) { - for (const comparator of comparators) { - if (comparator.operator === '>=' || comparator.operator === '>') { - if (!lowerBound || semver.gt(comparator.semver, lowerBound)) { - lowerBound = comparator.semver - } - } - } - } - } - - return lowerBound -} - async function assertPackage (name, version, dependencyVersionRange, external) { - // Apply version cap from latests.json if available - let cappedVersionRange = dependencyVersionRange - - // Handle simple major version numbers like "1" or exact versions like "1.0.0" - if (dependencyVersionRange) { - const simpleVersionMatch = dependencyVersionRange.match(/^(\d+)$/) - const exactVersionMatch = dependencyVersionRange.match(/^(\d+)\.(\d+)\.(\d+)$/) - - // Attempt to fix issue seen with fastify - if (simpleVersionMatch) { - // For simple major versions like "1", restrict to that major version only - const majorVersion = simpleVersionMatch[1] - cappedVersionRange = `>=${majorVersion}.0.0 <${parseInt(majorVersion) + 1}.0.0` - } else if (exactVersionMatch) { - // For exact versions like "1.0.0" or "2.2.2", use exactly that version - // Don't allow newer versions even within the same major - cappedVersionRange = dependencyVersionRange - } else if (latests.latests[name]) { - // Apply normal capping for other version ranges - const latestVersion = latests.latests[name] - if (typeof dependencyVersionRange === 'string') { - cappedVersionRange = applyCap(dependencyVersionRange, latestVersion) - } - } - } + const alreadyCapped = dependencyVersionRange.includes('-') + const cappedVersionRange = external || alreadyCapped + ? dependencyVersionRange + : `${dependencyVersionRange} <=${latests.latests[name]}` const dependencies = { [name]: cappedVersionRange } if (deps[name]) {