Skip to content

Commit 1cae620

Browse files
ruyadornoruyadorno
committed
rewrite benchmark suite
1 parent 075cd9b commit 1cae620

32 files changed

+1045
-609
lines changed

‎.editorconfig

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# EditorConfig is awesome: https://EditorConfig.org
2+
3+
# top-most EditorConfig file
4+
root = true
5+
6+
# All files
7+
[*]
8+
charset = utf-8
9+
end_of_line = lf
10+
insert_final_newline = true
11+
trim_trailing_whitespace = true
12+
13+
# Shell scripts
14+
[*.sh]
15+
indent_style = space
16+
indent_size = 2
17+
18+
# JavaScript/TypeScript files
19+
[*.{js,mjs,ts}]
20+
indent_style = space
21+
indent_size = 2
22+
23+
# JSON files
24+
[*.json]
25+
indent_style = space
26+
indent_size = 2
27+
28+
# CSS files
29+
[*.css]
30+
indent_style = space
31+
indent_size = 2
32+
33+
# HTML files
34+
[*.html]
35+
indent_style = space
36+
indent_size = 2
37+
# Markdown files
38+
[*.md]
39+
trim_trailing_whitespace = false
40+
41+
# Package.json files
42+
[package.json]
43+
indent_style = space
44+
indent_size = 2

‎.github/workflows/benchmark.yaml

Lines changed: 81 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -2,170 +2,110 @@ name: Package Manager Benchmarks
22
on:
33
push:
44
workflow_dispatch:
5+
inputs:
6+
fixtures:
7+
description: 'The fixture to run the benchmarks on'
8+
default: '["next", "astro", "svelte", "vue"]'
9+
variations:
10+
description: 'The benchmark variations to run'
11+
default: '["cache", "cache+lockfile", "cache+node_modules", "cache+lockfile+node_modules", "clean", "lockfile", "lockfile+node_modules", "node_modules"]'
12+
binaries:
13+
description: 'The binaries to run the benchmarks on'
14+
default: '"npm,yarn,berry,pnpm,vlt,bun,deno,nx,turbo,node"'
15+
warmup:
16+
description: 'The number of warmup runs on each benchmark'
17+
default: '2'
18+
runs:
19+
description: 'The number of runs on each benchmark'
20+
default: '10'
521
schedule:
6-
- cron: "0 0 * * *"
22+
- cron: "5 23 * * *"
723

824
# Prevent multiple runs from interfering with each other
925
concurrency:
1026
group: ${{ github.workflow }}-${{ github.ref }}
1127
cancel-in-progress: true
1228

1329
jobs:
14-
setup:
15-
name: 'Setup'
16-
runs-on: blacksmith-4vcpu-ubuntu-2404
17-
timeout-minutes: 10
18-
steps:
19-
- uses: actions/checkout@v4
20-
- name: Cache Bins
21-
uses: useblacksmith/cache@v5
22-
with:
23-
path: bins
24-
key: bins
25-
- name: Install Node
26-
uses: useblacksmith/setup-node@v5
27-
with:
28-
node-version: '22'
29-
- name: Install & Setup Tools
30-
run: |
31-
bash ./scripts/setup.sh
32-
3330
benchmark:
3431
name: 'Run Benchmarks'
35-
runs-on: blacksmith-4vcpu-ubuntu-2404
36-
needs: [setup]
32+
runs-on: ubuntu-latest
3733
timeout-minutes: 30
3834
strategy:
3935
matrix:
40-
project: [next, astro, svelte, vue]
41-
cache: [cold, warm]
36+
fixture: ${{ fromJson(inputs.fixtures || '["vue"]') }}
37+
variation: ${{ fromJson(inputs.variations || '["cache"]') }}
4238
include:
43-
- project: next
44-
cache: cold
45-
- project: astro
46-
cache: cold
47-
- project: svelte
48-
cache: cold
49-
- project: vue
50-
cache: cold
51-
- project: next
52-
cache: warm
53-
- project: astro
54-
cache: warm
55-
- project: svelte
56-
cache: warm
57-
- project: vue
58-
cache: warm
39+
- variation: "run"
40+
fixture: "run"
41+
env:
42+
BENCH_INCLUDE: ${{ fromJson(inputs.binaries || '"berry"') }}
43+
BENCH_WARMUP: ${{ inputs.warmup || '1' }}
44+
BENCH_RUNS: ${{ inputs.runs || '2' }}
5945
steps:
6046
- uses: actions/checkout@v4
6147
- name: Install Node
62-
uses: useblacksmith/setup-node@v5
48+
uses: actions/setup-node@v4
6349
with:
64-
node-version: '22'
65-
- name: Install System Dependencies
50+
node-version: '24'
51+
- name: Install & Setup Tools
6652
run: |
67-
sudo apt-get update
68-
sudo apt-get install -y jq hyperfine
69-
- name: Restore Bins
70-
uses: useblacksmith/cache/restore@v5
71-
with:
72-
path: bins
73-
key: bins
74-
- name: Run Project Benchmarks
53+
bash ./scripts/setup.sh
54+
- name: Run Benchmarks variations
7555
run: |
76-
if [ "${{ matrix.cache }}" = "warm" ]; then
77-
bash ./scripts/install-warm.sh ${{ matrix.project }}
56+
if [ "${{ matrix.variation }}" = "run" ]; then
57+
bash ./scripts/benchmark.sh run run
7858
else
79-
bash ./scripts/install.sh ${{ matrix.project }}
59+
bash ./scripts/benchmark.sh ${{ matrix.fixture }} ${{ matrix.variation }}
8060
fi
8161
- name: Upload Benchmark Results
8262
uses: actions/upload-artifact@v4
8363
with:
84-
name: ${{ matrix.project }}-${{ matrix.cache }}-results
85-
path: ./results/${{ matrix.project }}/
86-
retention-days: 7
87-
88-
task:
89-
name: 'Benchmark Running Tasks'
90-
runs-on: blacksmith-4vcpu-ubuntu-2404
91-
needs: [setup]
92-
timeout-minutes: 15
93-
steps:
94-
- uses: actions/checkout@v4
95-
- name: Install Node
96-
uses: useblacksmith/setup-node@v5
97-
with:
98-
node-version: '22'
99-
- name: Install System Dependencies
100-
run: |
101-
sudo apt-get update
102-
sudo apt-get install -y jq hyperfine
103-
- name: Restore Bins
104-
uses: useblacksmith/cache/restore@v5
105-
with:
106-
path: bins
107-
key: bins
108-
- name: Run Task Execution Benchmarks
109-
run: |
110-
bash ./scripts/run.sh
111-
- name: Upload Benchmark Results
112-
uses: actions/upload-artifact@v4
113-
with:
114-
name: task-benchmark-results
115-
path: ./results/run.json
64+
name: results-${{ matrix.fixture }}-${{ matrix.variation }}
65+
path: ./results/${{ matrix.fixture }}/${{ matrix.variation }}/
11666
retention-days: 7
117-
118-
process:
119-
name: 'Process Results'
120-
runs-on: blacksmith-4vcpu-ubuntu-2404
121-
needs: [benchmark, task]
122-
timeout-minutes: 5
123-
steps:
124-
- uses: actions/checkout@v4
125-
- name: Install Node
126-
uses: useblacksmith/setup-node@v5
127-
with:
128-
node-version: '22'
129-
- name: Download Results
130-
uses: actions/download-artifact@v4
131-
with:
132-
path: results
133-
merge-multiple: true
134-
- name: Process Results
135-
run: |
136-
bash ./scripts/process-results.sh
137-
- name: Upload Processed Results
138-
uses: actions/upload-artifact@v4
139-
with:
140-
name: processed-results
141-
path: chart/results/
142-
retention-days: 7
143-
144-
deploy:
145-
name: 'Deploy Results'
146-
runs-on: blacksmith-4vcpu-ubuntu-2404
147-
needs: [process]
148-
if: github.ref == 'refs/heads/main'
149-
steps:
150-
- uses: actions/checkout@v4
151-
- name: Download Results
152-
uses: actions/download-artifact@v4
153-
with:
154-
name: processed-results
155-
path: chart/results/
156-
- name: Deploy to GitHub Pages
157-
uses: peaceiris/actions-gh-pages@v3
158-
with:
159-
github_token: ${{ secrets.GITHUB_TOKEN }}
160-
publish_dir: chart/results
161-
force_orphan: true
162-
163-
cleanup:
164-
name: 'Cleanup'
165-
needs: [deploy]
166-
runs-on: blacksmith-4vcpu-ubuntu-2404
167-
if: always()
168-
steps:
169-
- name: Delete old artifacts
170-
run: |
171-
gh api repos/${{ github.repository }}/actions/artifacts --paginate | jq -r '.artifacts[] | select(.expired) | .id' | xargs -I {} gh api repos/${{ github.repository }}/actions/artifacts/{} -X DELETE
67+
# process:
68+
# name: 'Process Results'
69+
# runs-on: ubuntu-latest
70+
# needs: [benchmark]
71+
# timeout-minutes: 5
72+
# steps:
73+
# - uses: actions/checkout@v4
74+
# - name: Install Node
75+
# uses: actions/setup-node@v4
76+
# with:
77+
# node-version: '24'
78+
# - name: Download Results
79+
# uses: actions/download-artifact@v4
80+
# with:
81+
# path: results
82+
# pattern: results-*
83+
# - name: Process Results
84+
# run: |
85+
# bash ./scripts/process-results.sh
86+
# - name: Upload Processed Results
87+
# uses: actions/upload-artifact@v4
88+
# with:
89+
# name: charts
90+
# path: charts
91+
# retention-days: 7
92+
# deploy:
93+
# name: 'Deploy Results'
94+
# runs-on: ubuntu-latest
95+
# needs: [process]
96+
# permissions:
97+
# contents: write
98+
# # if: github.ref == 'refs/heads/main'
99+
# steps:
100+
# - uses: actions/checkout@v4
101+
# - name: Download Results
102+
# uses: actions/download-artifact@v4
103+
# with:
104+
# name: charts
105+
# path: charts/
106+
# - name: Deploy to GitHub Pages
107+
# uses: peaceiris/actions-gh-pages@v3
108+
# with:
109+
# github_token: ${{ secrets.GITHUB_TOKEN }}
110+
# publish_dir: charts
111+
# force_orphan: true

‎scripts/benchmark.sh

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Exit on error
2+
set -Eeuxo pipefail
3+
4+
# Check if fixture name is provided
5+
if [ -z "$1" ]; then
6+
echo "Error: Fixture name not provided"
7+
echo "Usage: $0 <fixture-name> <variation>"
8+
exit 1
9+
fi
10+
11+
# Check if variation is provided
12+
if [ -z "$2" ]; then
13+
echo "Error: Benchmark variation not provided"
14+
echo "Usage: $0 <fixture-name> <variation>"
15+
exit 1
16+
fi
17+
18+
# Navigate to the fixture directory
19+
pushd "./fixtures/$1"
20+
21+
# Run the install variation
22+
case "$2" in
23+
cache)
24+
bash ../../scripts/variations/cache.sh "../../scripts" "../../results" "$1" "$2"
25+
;;
26+
cache+lockfile)
27+
bash ../../scripts/variations/cache+lockfile.sh "../../scripts" "../../results" "$1" "$2"
28+
;;
29+
cache+node_modules)
30+
bash ../../scripts/variations/cache+node_modules.sh "../../scripts" "../../results" "$1" "$2"
31+
;;
32+
cache+lockfile+node_modules)
33+
bash ../../scripts/variations/cache+lockfile+node_modules.sh "../../scripts" "../../results" "$1" "$2"
34+
;;
35+
lockfile)
36+
bash ../../scripts/variations/lockfile.sh "../../scripts" "../../results" "$1" "$2"
37+
;;
38+
lockfile+node_modules)
39+
bash ../../scripts/variations/lockfile+node_modules.sh "../../scripts" "../../results" "$1" "$2"
40+
;;
41+
node_modules)
42+
bash ../../scripts/variations/node_modules.sh "../../scripts" "../../results" "$1" "$2"
43+
;;
44+
clean)
45+
bash ../../scripts/variations/clean.sh "../../scripts" "../../results" "$1" "$2"
46+
;;
47+
run)
48+
bash ../../scripts/variations/run.sh "../../scripts" "../../results" "run" "run"
49+
;;
50+
*)
51+
echo "Error: Unknown install variation '$2'"
52+
exit 1
53+
;;
54+
esac
55+
56+
popd
57+
echo "Installation benchmark suite completed successfully!"

0 commit comments

Comments
 (0)