Skip to content

Commit 5013ed8

Browse files
authored
add flakiness report workflow (#5856)
1 parent 784197d commit 5013ed8

File tree

6 files changed

+389
-0
lines changed

6 files changed

+389
-0
lines changed

.github/workflows/flakiness.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: Flakiness Report
2+
3+
on:
4+
schedule:
5+
- cron: '0 6 * * *'
6+
7+
jobs:
8+
flakiness:
9+
runs-on: ubuntu-latest
10+
permissions:
11+
actions: read
12+
env:
13+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
14+
steps:
15+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
16+
- uses: ./.github/actions/node
17+
with:
18+
version: ''
19+
- run: node scripts/flakiness.mjs

LICENSE-3rdparty.csv

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ dev,mocha,MIT,Copyright 2011-2018 JS Foundation and contributors https://js.foun
5959
dev,multer,MIT,Copyright 2014 Hage Yaapa
6060
dev,nock,MIT,Copyright 2017 Pedro Teixeira and other contributors
6161
dev,nyc,ISC,Copyright 2015 Contributors
62+
dev,octokit,MIT,Copyright 2023 Octokit contributors
6263
dev,proxyquire,MIT,Copyright 2013 Thorsten Lorenz
6364
dev,rimraf,ISC,Copyright Isaac Z. Schlueter and Contributors
6465
dev,semver,ISC,Copyright Isaac Z. Schlueter and Contributors

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@
146146
"multer": "^2.0.1",
147147
"nock": "^11.9.1",
148148
"nyc": "^15.1.0",
149+
"octokit": "^5.0.3",
149150
"proxyquire": "^1.8.0",
150151
"rimraf": "^3.0.2",
151152
"semver": "^7.7.2",

scripts/flakiness.mjs

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
'use strict'
2+
3+
/* eslint-disable no-console */
4+
5+
import { Octokit } from 'octokit'
6+
7+
const { DAYS = '1' } = process.env
8+
9+
const ONE_DAY = 24 * 60 * 60 * 1000
10+
11+
const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN })
12+
const workflows = [
13+
'.github/workflows/apm-capabilities.yml',
14+
'.github/workflows/apm-integrations.yml',
15+
'.github/workflows/appsec.yml',
16+
'.github/workflows/debugger.yml',
17+
'.github/workflows/llmobs.yml',
18+
'.github/workflows/platform.yml',
19+
'.github/workflows/project.yml',
20+
'.github/workflows/system-tests.yml',
21+
'.github/workflows/test-optimization.yml'
22+
]
23+
24+
const flaky = {}
25+
26+
async function checkWorkflowRuns (id, page = 1) {
27+
const response = await octokit.rest.actions.listWorkflowRuns({
28+
owner: 'DataDog',
29+
repo: 'dd-trace-js',
30+
page,
31+
per_page: 100, // max is 100
32+
status: 'success',
33+
workflow_id: id
34+
})
35+
36+
const runs = response.data.workflow_runs
37+
38+
if (runs.length === 0) return
39+
40+
const promises = []
41+
42+
for (const run of runs) {
43+
if (run.run_attempt === 1) continue
44+
if (Date.parse(run.created_at) < Date.now() - DAYS * ONE_DAY) {
45+
return Promise.all(promises)
46+
}
47+
48+
promises.push(checkWorkflowJobs(run.id))
49+
}
50+
51+
promises.push(checkWorkflowRuns(id, page + 1))
52+
53+
return Promise.all(promises)
54+
}
55+
56+
async function checkWorkflowJobs (id, page = 1) {
57+
const response = await octokit.rest.actions.listJobsForWorkflowRunAttempt({
58+
attempt_number: 1, // ignore other attempts to keep things simple
59+
owner: 'DataDog',
60+
repo: 'dd-trace-js',
61+
run_id: id,
62+
page,
63+
per_page: 100 // max is 100
64+
})
65+
66+
const { jobs } = response.data
67+
68+
if (jobs.length === 0) return
69+
70+
for (const job of jobs) {
71+
if (job.conclusion !== 'failure') continue
72+
73+
const workflow = job.workflow_name
74+
75+
flaky[workflow] = flaky[workflow] || {}
76+
flaky[workflow][job.name] = flaky[workflow][job.name] || []
77+
flaky[workflow][job.name].push(job.html_url)
78+
}
79+
80+
return checkWorkflowJobs(id, page + 1)
81+
}
82+
83+
await Promise.all(workflows.map(w => checkWorkflowRuns(w)))
84+
85+
// TODO: Report this somewhere useful instead.
86+
if (Object.keys(flaky).length === 0) {
87+
console.log(`*No flaky jobs seen in the last ${DAYS > 1 ? `${DAYS} days` : 'day'}*`)
88+
} else {
89+
console.log(`*Flaky jobs seen in the last ${DAYS > 1 ? `${DAYS} days` : 'day'}*`)
90+
for (const [workflow, jobs] of Object.entries(flaky).sort()) {
91+
console.log(`* ${workflow}`)
92+
for (const [job, urls] of Object.entries(jobs).sort()) {
93+
console.log(` * ${job}`)
94+
for (const url of urls.sort()) {
95+
console.log(` * ${url}`)
96+
}
97+
}
98+
}
99+
}

scripts/verify-ci-config.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ checkPlugins(path.join(__dirname, '..', '.github', 'workflows', 'test-optimizati
161161
const IGNORED_WORKFLOWS = {
162162
all: [
163163
'codeql-analysis.yml',
164+
'flakiness.yml',
164165
'pr-labels.yml',
165166
'release-3.yml',
166167
'release-4.yml',

0 commit comments

Comments
 (0)