diff --git a/.github/workflows/auto-bump-test-package-versions.yml b/.github/workflows/auto-bump-test-package-versions.yml new file mode 100644 index 00000000000..b7d5dfbf5cb --- /dev/null +++ b/.github/workflows/auto-bump-test-package-versions.yml @@ -0,0 +1,53 @@ +name: Auto bump latest tested NPM packages + +on: + schedule: + - cron: '0 0 * * 0' # Every Sunday at midnight + workflow_dispatch: + +jobs: + bump_package_versions: + runs-on: ubuntu-latest + permissions: + actions: read # read secrets + contents: write # Creates a branch + pull-requests: write # Creates a PR + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Setup Node.js + uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # v4.3.0 + with: + node-version: '18' + + - name: Install dependencies + run: yarn install + + - name: Update package versions in latests.json + run: node scripts/outdated.js + + - name: Create Pull Request + id: pr + uses: peter-evans/create-pull-request@dd2324fc52d5d43c699a5636bcf19fceaa70c284 # v7.0.7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: "bot/test-package-versions-bump" + commit-message: "[Test Package Versions Bump]" + delete-branch: true + base: master + title: "[Test Package Versions Bump] Updating package versions" + labels: dependencies + body: | + Automated update of the latest versions of NPM packages used in tests. + + # If desired we can enable automerge with the necessary PAT created + # - name: Enable Pull Request Automerge + # if: steps.pr.outputs.pull-request-operation == 'created' + # uses: peter-evans/enable-pull-request-automerge@a660677d5469627102a1c1e11409dd063606628d # v3.0.0 + # with: + # token: ${{ secrets.GHA_PAT }} + # pull-request-number: ${{ steps.pr.outputs.pull-request-number }} diff --git a/packages/datadog-instrumentations/src/helpers/latests.json b/packages/datadog-instrumentations/src/helpers/latests.json new file mode 100644 index 00000000000..9898f34982c --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/latests.json @@ -0,0 +1,116 @@ +{ + "pinned": [ + "fastify", + "express" + ], + "onlyUseLatestTag": [], + "ignored": [ + "aerospike", + "mariadb", + "microgateway-core", + "winston" + ], + "latests": { + "amqp10": "3.6.0", + "amqplib": "0.10.7", + "apollo-server-core": "3.13.0", + "@apollo/server": "4.11.3", + "@apollo/gateway": "2.10.0", + "avsc": "5.7.7", + "@smithy/smithy-client": "4.2.0", + "@aws-sdk/smithy-client": "3.374.0", + "aws-sdk": "2.1692.0", + "@azure/functions": "4.7.0", + "bluebird": "3.7.2", + "body-parser": "2.2.0", + "bunyan": "2.0.5", + "cassandra-driver": "4.8.0", + "connect": "3.7.0", + "cookie-parser": "1.4.7", + "cookie": "1.0.2", + "couchbase": "4.4.5", + "@cucumber/cucumber": "11.2.0", + "cypress": "14.2.1", + "@elastic/transport": "8.9.5", + "@elastic/elasticsearch": "8.17.1", + "elasticsearch": "16.7.3", + "express-mongo-sanitize": "2.2.0", + "express-session": "1.18.1", + "express": "5.0.1", + "fastify": "4.28.1", + "find-my-way": "9.3.0", + "fs": "0.0.2", + "generic-pool": "3.9.0", + "@google-cloud/pubsub": "4.11.0", + "@graphql-tools/executor": "1.4.7", + "graphql": "16.10.0", + "@grpc/grpc-js": "1.13.2", + "handlebars": "4.7.8", + "@hapi/hapi": "21.4.0", + "hapi": "18.1.0", + "ioredis": "5.6.0", + "jest-environment-node": "29.7.0", + "jest-environment-jsdom": "29.7.0", + "@jest/core": "29.7.0", + "@jest/test-sequencer": "29.7.0", + "@jest/reporters": "29.7.0", + "jest-circus": "29.7.0", + "@jest/transform": "29.7.0", + "jest-config": "29.7.0", + "jest-runtime": "29.7.0", + "jest-worker": "29.7.0", + "kafkajs": "2.2.4", + "knex": "3.1.0", + "koa": "2.16.0", + "@koa/router": "13.1.0", + "koa-router": "13.0.1", + "@langchain/core": "0.3.43", + "@langchain/openai": "0.5.2", + "ldapjs": "3.0.7", + "limitd-client": "2.14.1", + "lodash": "4.17.21", + "memcached": "2.2.2", + "moleculer": "0.14.35", + "mongodb-core": "3.2.7", + "mongodb": "6.15.0", + "mongoose": "8.13.2", + "mquery": "5.0.0", + "multer": "1.4.5-lts.2", + "mysql": "2.18.1", + "mysql2": "3.14.0", + "next": "15.2.4", + "node-serialize": "0.0.4", + "nyc": "17.1.0", + "openai": "4.91.1", + "@opensearch-project/opensearch": "3.5.0", + "oracledb": "6.8.0", + "paperplane": "3.1.2", + "passport-http": "0.3.0", + "passport-local": "1.0.0", + "passport": "0.7.0", + "pg": "8.14.1", + "pino": "9.6.0", + "pino-pretty": "13.0.0", + "@playwright/test": "1.51.1", + "playwright": "1.51.1", + "promise-js": "0.0.7", + "promise": "8.3.0", + "protobufjs": "7.4.0", + "pug": "3.0.3", + "q": "2.0.3", + "@node-redis/client": "1.0.6", + "@redis/client": "1.6.0", + "redis": "4.7.0", + "restify": "11.1.0", + "rhea": "3.0.4", + "router": "2.2.0", + "selenium-webdriver": "4.30.0", + "sequelize": "6.37.7", + "sharedb": "5.2.1", + "tedious": "19.0.0", + "undici": "7.7.0", + "vitest": "3.1.1", + "@vitest/runner": "3.1.1", + "when": "3.7.8" + } +} diff --git a/scripts/helpers/versioning.js b/scripts/helpers/versioning.js new file mode 100644 index 00000000000..b51eaafee8b --- /dev/null +++ b/scripts/helpers/versioning.js @@ -0,0 +1,78 @@ +const fs = require('fs') +const path = require('path') +const childProcess = require('child_process') +const proxyquire = require('proxyquire') + +const versionLists = {} +const names = [] + +const filter = process.env.hasOwnProperty('PLUGINS') && process.env.PLUGINS.split('|') + +fs.readdirSync(path.join(__dirname, '../../packages/datadog-instrumentations/src')) + .filter(file => file.endsWith('js')) + .forEach(file => { + file = file.replace('.js', '') + + if (!filter || filter.includes(file)) { + names.push(file) + } + }) + +async function getVersionList (name) { + if (versionLists[name]) { + return versionLists[name] + } + const list = await npmView(`${name} versions`) + versionLists[name] = list + return list +} + +function npmView (input) { + return new Promise((resolve, reject) => { + childProcess.exec(`npm view ${input} --json`, (err, stdout) => { + if (err) { + reject(err) + return + } + resolve(JSON.parse(stdout.toString('utf8'))) + }) + }) +} + +function loadInstFile (file, instrumentations) { + const instrument = { + addHook (instrumentation) { + instrumentations.push(instrumentation) + } + } + + const instPath = path.join(__dirname, `../../packages/datadog-instrumentations/src/${file}`) + + proxyquire.noPreserveCache()(instPath, { + './helpers/instrument': instrument, + '../helpers/instrument': instrument + }) +} + +function getInternals () { + return names.map(key => { + const instrumentations = [] + const name = key + + try { + loadInstFile(`${name}/server.js`, instrumentations) + loadInstFile(`${name}/client.js`, instrumentations) + } catch (e) { + loadInstFile(`${name}.js`, instrumentations) + } + + return instrumentations + }).reduce((prev, next) => prev.concat(next), []) +} + +module.exports = { + getVersionList, + npmView, + loadInstFile, + getInternals +} diff --git a/scripts/install_plugin_modules.js b/scripts/install_plugin_modules.js index cfcc9b29371..929106e9c13 100644 --- a/scripts/install_plugin_modules.js +++ b/scripts/install_plugin_modules.js @@ -9,6 +9,7 @@ const exec = require('./helpers/exec') const childProcess = require('child_process') const externals = require('../packages/dd-trace/test/plugins/externals') const { getInstrumentation } = require('../packages/dd-trace/test/setup/helpers/load-inst') +const latests = require('../packages/datadog-instrumentations/src/helpers/latests.json') const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json') @@ -106,10 +107,16 @@ function assertFolder (name, version) { } async function assertPackage (name, version, dependencyVersionRange, external) { - const dependencies = { [name]: dependencyVersionRange } + const alreadyCapped = dependencyVersionRange.includes('-') + const cappedVersionRange = external || alreadyCapped + ? dependencyVersionRange + : `${dependencyVersionRange} <=${latests.latests[name]}` + + const dependencies = { [name]: cappedVersionRange } if (deps[name]) { - await addDependencies(dependencies, name, dependencyVersionRange) + await addDependencies(dependencies, name, cappedVersionRange) } + const pkg = { name: [name, sha1(name).substr(0, 8), sha1(version)].filter(val => val).join('-'), version: '1.0.0', @@ -131,7 +138,6 @@ async function assertPackage (name, version, dependencyVersionRange, external) { } fs.writeFileSync(filename(name, version, 'package.json'), JSON.stringify(pkg, null, 2) + '\n') } - async function addDependencies (dependencies, name, versionRange) { const versionList = await getVersionList(name) const version = semver.maxSatisfying(versionList, versionRange) diff --git a/scripts/outdated.js b/scripts/outdated.js new file mode 100644 index 00000000000..91789855108 --- /dev/null +++ b/scripts/outdated.js @@ -0,0 +1,188 @@ +/* eslint-disable no-console */ +'use strict' + +const { getInternals } = require('./helpers/versioning') +const path = require('path') +const fs = require('fs') +const semver = require('semver') +const childProcess = require('child_process') + +const latestsPath = path.join( + __dirname, + '..', + 'packages', + 'datadog-instrumentations', + 'src', + 'helpers', + 'latests.json' +) + +const internalsNames = Array.from(new Set(getInternals().map(n => n.name))) + .filter(x => typeof x === 'string' && x !== 'child_process' && !x.startsWith('node:')) + +// Packages that should be ignored during version checking - these won't be included in latests.json +const IGNORED_PACKAGES = [ + // Add package names here + 'aerospike', // I think this is due to architecture issues? + 'mariadb', // mariadb esm tests were failing + 'microgateway-core', // 'microgateway-core' was failing to find a directory + 'winston' // winston esm tests were failing +] + +// Packages that should be pinned to specific versions +const PINNED_PACKAGES = { + // Example: 'express': '4.17.3' + fastify: '4.28.1', // v5+ is not supported + express: '4.21.2' +} + +// Packages that should only use the 'latest' tag (not 'next' or other dist-tags) +// Some packages have a next tag that is a stable semver version +const ONLY_USE_LATEST_TAG = [ + // Example: 'router' +] + +// Initial structure for latests.json that will be recreated each run +const outputData = { + pinned: Object.keys(PINNED_PACKAGES), + onlyUseLatestTag: ONLY_USE_LATEST_TAG, + ignored: IGNORED_PACKAGES, + latests: {} +} + +function npmView (input) { + return new Promise((resolve, reject) => { + childProcess.exec(`npm view ${input} --json`, (err, stdout) => { + if (err) { + reject(err) + return + } + try { + resolve(JSON.parse(stdout.toString('utf8'))) + } catch (e) { + reject(new Error(`Failed to parse npm output for ${input}: ${e.message}`)) + } + }) + }) +} + +async function getHighestCompatibleVersion (name) { + try { + if (IGNORED_PACKAGES.includes(name)) { + console.log(`Skipping "${name}" as it's in the ignored list`) + return null + } + + // If package is hardcoded as pinned, return the pinned version but also check latest + // this is for logging purposes + if (PINNED_PACKAGES[name]) { + const pinnedVersion = PINNED_PACKAGES[name] + + try { + const distTags = await npmView(`${name} dist-tags`) + const latestTagged = distTags.latest + + if (latestTagged && semver.gt(latestTagged, pinnedVersion)) { + console.log(`Note: "${name}" is pinned to ${pinnedVersion}, but ${latestTagged} is available`) + } + } catch (err) { + // Just log the error but continue with the pinned version + console.log(`Warning: Could not fetch latest version for pinned package "${name}": ${err.message}`) + } + + return pinnedVersion + } + + // ideally we can just use `latest` tag, but a few use `next` + const distTags = await npmView(`${name} dist-tags`) + + // Get the latest tagged version + const latestTagged = distTags.latest + + if (!latestTagged) { + console.log(`Warning: Could not fetch latest version for "${name}"`) + return null + } + + // If package is in the onlyUseLatestTag list, always use the 'latest' tag + if (ONLY_USE_LATEST_TAG.includes(name)) { + return latestTagged + } + + // Get all available versions + const allVersions = await npmView(`${name} versions`) + + // Find the highest non-prerelease version available + const stableVersions = allVersions.filter(v => !semver.prerelease(v)) + const highestStableVersion = stableVersions.sort(semver.compare).pop() + + // Use the highest stable version if it's greater than the latest tag + if (highestStableVersion && semver.gt(highestStableVersion, latestTagged)) { + process.stdout.write(` found version ${highestStableVersion} (higher than 'latest' tag ${latestTagged})`) + return highestStableVersion + } + + return latestTagged + } catch (error) { + console.error(`Error fetching version for "${name}":`, error.message) + return null + } +} + +async function fix () { + console.log('Starting fix operation...') + console.log(`Found ${internalsNames.length} packages to process`) + + const latests = {} + let processed = 0 + const total = internalsNames.length + + for (const name of internalsNames) { + processed++ + process.stdout.write(`Processing package ${processed}/${total}: ${name}...`) + + // Skip ignored packages + if (IGNORED_PACKAGES.includes(name)) { + process.stdout.write(' IGNORED\n') + continue + } + + try { + // Handle hardcoded pinned packages + if (PINNED_PACKAGES[name]) { + const pinnedVersion = PINNED_PACKAGES[name] + latests[name] = pinnedVersion + process.stdout.write(` PINNED to version ${pinnedVersion}\n`) + continue + } + + // Normal package processing + const latestVersion = await getHighestCompatibleVersion(name) + if (latestVersion) { + latests[name] = latestVersion + process.stdout.write(` found version ${latestVersion}\n`) + } else { + process.stdout.write(' WARNING: no version found\n') + } + } catch (error) { + process.stdout.write(' ERROR\n') + console.error(`Error processing "${name}":`, error.message) + } + } + + // Update the output data + outputData.latests = latests + + // Write the updated configuration with a comment at the top + console.log('\nWriting updated versions to latests.json...') + + // Convert to JSON with proper indentation + const jsonContent = JSON.stringify(outputData, null, 2) + + fs.writeFileSync(latestsPath, jsonContent) + + console.log('Successfully updated latests.json') + console.log(`Processed ${total} packages`) +} + +fix()