Skip to content

[WIP] Implement basic CI version capper #5346

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 15 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions .github/workflows/auto-bump-test-package-versions.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
name: Auto bump latest tested NPM packages

on:
schedule:
- cron: '0 0 * * 0' # Every Sunday at midnight
workflow_dispatch:

jobs:
bump_package_versions:
runs-on: ubuntu-latest
permissions:
actions: read # read secrets
contents: write # Creates a branch
pull-requests: write # Creates a PR
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- name: Setup Node.js
uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # v4.3.0
with:
node-version: '18'

- name: Install dependencies
run: yarn install

- name: Update package versions in latests.json
run: node scripts/outdated.js

- name: Create Pull Request
id: pr
uses: peter-evans/create-pull-request@dd2324fc52d5d43c699a5636bcf19fceaa70c284 # v7.0.7
with:
token: ${{ secrets.GITHUB_TOKEN }}
branch: "bot/test-package-versions-bump"
commit-message: "[Test Package Versions Bump]"
delete-branch: true
base: master
title: "[Test Package Versions Bump] Updating package versions"
labels: dependencies
body: |
Automated update of the latest versions of NPM packages used in tests.

# If desired we can enable automerge with the necessary PAT created
# - name: Enable Pull Request Automerge
# if: steps.pr.outputs.pull-request-operation == 'created'
# uses: peter-evans/enable-pull-request-automerge@a660677d5469627102a1c1e11409dd063606628d # v3.0.0
# with:
# token: ${{ secrets.GHA_PAT }}
# pull-request-number: ${{ steps.pr.outputs.pull-request-number }}
116 changes: 116 additions & 0 deletions packages/datadog-instrumentations/src/helpers/latests.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
{
"pinned": [
"fastify",
"express"
],
"onlyUseLatestTag": [],
"ignored": [
"aerospike",
"mariadb",
"microgateway-core",
"winston"
],
"latests": {
"amqp10": "3.6.0",
"amqplib": "0.10.7",
"apollo-server-core": "3.13.0",
"@apollo/server": "4.11.3",
"@apollo/gateway": "2.10.0",
"avsc": "5.7.7",
"@smithy/smithy-client": "4.2.0",
"@aws-sdk/smithy-client": "3.374.0",
"aws-sdk": "2.1692.0",
"@azure/functions": "4.7.0",
"bluebird": "3.7.2",
"body-parser": "2.2.0",
"bunyan": "2.0.5",
"cassandra-driver": "4.8.0",
"connect": "3.7.0",
"cookie-parser": "1.4.7",
"cookie": "1.0.2",
"couchbase": "4.4.5",
"@cucumber/cucumber": "11.2.0",
"cypress": "14.2.1",
"@elastic/transport": "8.9.5",
"@elastic/elasticsearch": "8.17.1",
"elasticsearch": "16.7.3",
"express-mongo-sanitize": "2.2.0",
"express-session": "1.18.1",
"express": "5.0.1",
"fastify": "4.28.1",
"find-my-way": "9.3.0",
"fs": "0.0.2",
"generic-pool": "3.9.0",
"@google-cloud/pubsub": "4.11.0",
"@graphql-tools/executor": "1.4.7",
"graphql": "16.10.0",
"@grpc/grpc-js": "1.13.2",
"handlebars": "4.7.8",
"@hapi/hapi": "21.4.0",
"hapi": "18.1.0",
"ioredis": "5.6.0",
"jest-environment-node": "29.7.0",
"jest-environment-jsdom": "29.7.0",
"@jest/core": "29.7.0",
"@jest/test-sequencer": "29.7.0",
"@jest/reporters": "29.7.0",
"jest-circus": "29.7.0",
"@jest/transform": "29.7.0",
"jest-config": "29.7.0",
"jest-runtime": "29.7.0",
"jest-worker": "29.7.0",
"kafkajs": "2.2.4",
"knex": "3.1.0",
"koa": "2.16.0",
"@koa/router": "13.1.0",
"koa-router": "13.0.1",
"@langchain/core": "0.3.43",
"@langchain/openai": "0.5.2",
"ldapjs": "3.0.7",
"limitd-client": "2.14.1",
"lodash": "4.17.21",
"memcached": "2.2.2",
"moleculer": "0.14.35",
"mongodb-core": "3.2.7",
"mongodb": "6.15.0",
"mongoose": "8.13.2",
"mquery": "5.0.0",
"multer": "1.4.5-lts.2",
"mysql": "2.18.1",
"mysql2": "3.14.0",
"next": "15.2.4",
"node-serialize": "0.0.4",
"nyc": "17.1.0",
"openai": "4.91.1",
"@opensearch-project/opensearch": "3.5.0",
"oracledb": "6.8.0",
"paperplane": "3.1.2",
"passport-http": "0.3.0",
"passport-local": "1.0.0",
"passport": "0.7.0",
"pg": "8.14.1",
"pino": "9.6.0",
"pino-pretty": "13.0.0",
"@playwright/test": "1.51.1",
"playwright": "1.51.1",
"promise-js": "0.0.7",
"promise": "8.3.0",
"protobufjs": "7.4.0",
"pug": "3.0.3",
"q": "2.0.3",
"@node-redis/client": "1.0.6",
"@redis/client": "1.6.0",
"redis": "4.7.0",
"restify": "11.1.0",
"rhea": "3.0.4",
"router": "2.2.0",
"selenium-webdriver": "4.30.0",
"sequelize": "6.37.7",
"sharedb": "5.2.1",
"tedious": "19.0.0",
"undici": "7.7.0",
"vitest": "3.1.1",
"@vitest/runner": "3.1.1",
"when": "3.7.8"
}
}
78 changes: 78 additions & 0 deletions scripts/helpers/versioning.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
const fs = require('fs')
const path = require('path')
const childProcess = require('child_process')
const proxyquire = require('proxyquire')

const versionLists = {}
const names = []

const filter = process.env.hasOwnProperty('PLUGINS') && process.env.PLUGINS.split('|')

fs.readdirSync(path.join(__dirname, '../../packages/datadog-instrumentations/src'))
.filter(file => file.endsWith('js'))
.forEach(file => {
file = file.replace('.js', '')

if (!filter || filter.includes(file)) {
names.push(file)
}
})

async function getVersionList (name) {
if (versionLists[name]) {
return versionLists[name]
}
const list = await npmView(`${name} versions`)
versionLists[name] = list
return list
}

function npmView (input) {
return new Promise((resolve, reject) => {
childProcess.exec(`npm view ${input} --json`, (err, stdout) => {
if (err) {
reject(err)
return
}
resolve(JSON.parse(stdout.toString('utf8')))
})
})
}

function loadInstFile (file, instrumentations) {
const instrument = {
addHook (instrumentation) {
instrumentations.push(instrumentation)
}
}

const instPath = path.join(__dirname, `../../packages/datadog-instrumentations/src/${file}`)

proxyquire.noPreserveCache()(instPath, {
'./helpers/instrument': instrument,
'../helpers/instrument': instrument
})
}

function getInternals () {
return names.map(key => {
const instrumentations = []
const name = key

try {
loadInstFile(`${name}/server.js`, instrumentations)
loadInstFile(`${name}/client.js`, instrumentations)
} catch (e) {
loadInstFile(`${name}.js`, instrumentations)
}

return instrumentations
}).reduce((prev, next) => prev.concat(next), [])
}

module.exports = {
getVersionList,
npmView,
loadInstFile,
getInternals
}
12 changes: 9 additions & 3 deletions scripts/install_plugin_modules.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const exec = require('./helpers/exec')
const childProcess = require('child_process')
const externals = require('../packages/dd-trace/test/plugins/externals')
const { getInstrumentation } = require('../packages/dd-trace/test/setup/helpers/load-inst')
const latests = require('../packages/datadog-instrumentations/src/helpers/latests.json')

const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json')

Expand Down Expand Up @@ -106,10 +107,16 @@ function assertFolder (name, version) {
}

async function assertPackage (name, version, dependencyVersionRange, external) {
const dependencies = { [name]: dependencyVersionRange }
const alreadyCapped = dependencyVersionRange.includes('-')
const cappedVersionRange = external || alreadyCapped
? dependencyVersionRange
: `${dependencyVersionRange} <=${latests.latests[name]}`

const dependencies = { [name]: cappedVersionRange }
if (deps[name]) {
await addDependencies(dependencies, name, dependencyVersionRange)
await addDependencies(dependencies, name, cappedVersionRange)
}

const pkg = {
name: [name, sha1(name).substr(0, 8), sha1(version)].filter(val => val).join('-'),
version: '1.0.0',
Expand All @@ -131,7 +138,6 @@ async function assertPackage (name, version, dependencyVersionRange, external) {
}
fs.writeFileSync(filename(name, version, 'package.json'), JSON.stringify(pkg, null, 2) + '\n')
}

async function addDependencies (dependencies, name, versionRange) {
const versionList = await getVersionList(name)
const version = semver.maxSatisfying(versionList, versionRange)
Expand Down
Loading