rewrite benchmark suite

Author: ruyadorno

.editorconfig

@@ -0,0 +1,44 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# All files
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+# Shell scripts
+[*.sh]
+indent_style = space
+indent_size = 2
+
+# JavaScript/TypeScript files
+[*.{js,mjs,ts}]
+indent_style = space
+indent_size = 2
+
+# JSON files
+[*.json]
+indent_style = space
+indent_size = 2
+
+# CSS files
+[*.css]
+indent_style = space
+indent_size = 2
+
+# HTML files
+[*.html]
+indent_style = space
+indent_size = 2
+# Markdown files
+[*.md]
+trim_trailing_whitespace = false
+
+# Package.json files
+[package.json]
+indent_style = space
+indent_size = 2 

.github/workflows/benchmark.yaml

@@ -2,170 +2,110 @@ name: Package Manager Benchmarks
 on: 
   push:
   workflow_dispatch:
+    inputs:
+      fixtures:
+        description: 'The fixture to run the benchmarks on'
+        default: '["next", "astro", "svelte", "vue"]'
+      variations:
+        description: 'The benchmark variations to run'
+        default: '["cache", "cache+lockfile", "cache+node_modules", "cache+lockfile+node_modules", "clean", "lockfile", "lockfile+node_modules", "node_modules"]'
+      binaries:
+        description: 'The binaries to run the benchmarks on'
+        default: '"npm,yarn,berry,pnpm,vlt,bun,deno,nx,turbo,node"'
+      warmup:
+        description: 'The number of warmup runs on each benchmark'
+        default: '2'
+      runs:
+        description: 'The number of runs on each benchmark'
+        default: '10'
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "5 23 * * *"
 
 # Prevent multiple runs from interfering with each other
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  setup: 
-    name: 'Setup'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v4
-      - name: Cache Bins
-        uses: useblacksmith/cache@v5
-        with:
-          path: bins
-          key: bins
-      - name: Install Node
-        uses: useblacksmith/setup-node@v5
-        with:
-          node-version: '22'
-      - name: Install & Setup Tools
-        run: |
-          bash ./scripts/setup.sh
-
   benchmark:
     name: 'Run Benchmarks'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [setup]
+    runs-on: ubuntu-latest
     timeout-minutes: 30
     strategy:
       matrix:
-        project: [next, astro, svelte, vue]
-        cache: [cold, warm]
+        fixture: ${{ fromJson(inputs.fixtures || '["vue", "svelte"]') }}
+        variation: ${{ fromJson(inputs.variations || '["cache", "cache+lockfile", "cache+node_modules", "cache+lockfile+node_modules", "clean", "lockfile", "lockfile+node_modules", "node_modules"]') }}
         include:
-          - project: next
-            cache: cold
-          - project: astro
-            cache: cold
-          - project: svelte
-            cache: cold
-          - project: vue
-            cache: cold
-          - project: next
-            cache: warm
-          - project: astro
-            cache: warm
-          - project: svelte
-            cache: warm
-          - project: vue
-            cache: warm
+          - variation: "run"
+            fixture: "run"
+    env:
+      BENCH_INCLUDE: ${{ fromJson(inputs.binaries || '"npm,yarn,berry,pnpm,vlt,bun,deno,node"') }}
+      BENCH_WARMUP: ${{ inputs.warmup || '1' }}
+      BENCH_RUNS: ${{ inputs.runs || '2' }}
     steps:
       - uses: actions/checkout@v4
       - name: Install Node
-        uses: useblacksmith/setup-node@v5
+        uses: actions/setup-node@v4
         with:
-          node-version: '22'
-      - name: Install System Dependencies
+          node-version: '24'
+      - name: Install & Setup Tools
         run: |
-          sudo apt-get update
-          sudo apt-get install -y jq hyperfine
-      - name: Restore Bins
-        uses: useblacksmith/cache/restore@v5
-        with:
-          path: bins
-          key: bins
-      - name: Run Project Benchmarks
+          bash ./scripts/setup.sh
+      - name: Run Benchmarks variations
         run: |
-          if [ "${{ matrix.cache }}" = "warm" ]; then
-            bash ./scripts/install-warm.sh ${{ matrix.project }}
+          if [ "${{ matrix.variation }}" = "run" ]; then
+            bash ./scripts/benchmark.sh run run
           else
-            bash ./scripts/install.sh ${{ matrix.project }}
+            bash ./scripts/benchmark.sh ${{ matrix.fixture }} ${{ matrix.variation }}
           fi
       - name: Upload Benchmark Results
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.project }}-${{ matrix.cache }}-results
-          path: ./results/${{ matrix.project }}/
+          name: results-${{ matrix.fixture }}-${{ matrix.variation }}
+          path: ./results/${{ matrix.fixture }}/${{ matrix.variation }}/
           retention-days: 7
-
-  task:
-    name: 'Benchmark Running Tasks'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [setup]
-    timeout-minutes: 15
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install Node
-        uses: useblacksmith/setup-node@v5
-        with:
-          node-version: '22'
-      - name: Install System Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y jq hyperfine
-      - name: Restore Bins
-        uses: useblacksmith/cache/restore@v5
-        with:
-          path: bins
-          key: bins
-      - name: Run Task Execution Benchmarks
-        run: |
-          bash ./scripts/run.sh
-      - name: Upload Benchmark Results
-        uses: actions/upload-artifact@v4
-        with:
-          name: task-benchmark-results
-          path: ./results/run.json
-          retention-days: 7
-
   process:
     name: 'Process Results'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    needs: [benchmark, task]
+    runs-on: ubuntu-latest
+    needs: [benchmark]
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@v4
       - name: Install Node
-        uses: useblacksmith/setup-node@v5
+        uses: actions/setup-node@v4
         with:
-          node-version: '22'
+          node-version: '24'
       - name: Download Results
         uses: actions/download-artifact@v4
         with:
           path: results
-          merge-multiple: true
+          pattern: results-*
       - name: Process Results
         run: |
           bash ./scripts/process-results.sh
       - name: Upload Processed Results
         uses: actions/upload-artifact@v4
         with:
-          name: processed-results
-          path: chart/results/
+          name: charts
+          path: charts
           retention-days: 7
-
   deploy:
     name: 'Deploy Results'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     needs: [process]
-    if: github.ref == 'refs/heads/main'
+    permissions:
+      contents: write
+    # if: github.ref == 'refs/heads/main'
     steps:
       - uses: actions/checkout@v4
       - name: Download Results
         uses: actions/download-artifact@v4
         with:
-          name: processed-results
-          path: chart/results/
+          name: charts
+          path: charts/
       - name: Deploy to GitHub Pages
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: chart/results
-          force_orphan: true
-
-  cleanup:
-    name: 'Cleanup'
-    needs: [deploy]
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    if: always()
-    steps:
-      - name: Delete old artifacts
-        run: |
-          gh api repos/${{ github.repository }}/actions/artifacts --paginate | jq -r '.artifacts[] | select(.expired) | .id' | xargs -I {} gh api repos/${{ github.repository }}/actions/artifacts/{} -X DELETE
+          publish_dir: charts
+          force_orphan: true

scripts/benchmark.sh

@@ -0,0 +1,57 @@
+# Exit on error
+set -Eeuxo pipefail
+
+# Check if fixture name is provided
+if [ -z "$1" ]; then
+    echo "Error: Fixture name not provided"
+    echo "Usage: $0 <fixture-name> <variation>"
+    exit 1
+fi
+
+# Check if variation is provided
+if [ -z "$2" ]; then
+    echo "Error: Benchmark variation not provided"
+    echo "Usage: $0 <fixture-name> <variation>"
+    exit 1
+fi
+
+# Navigate to the fixture directory
+pushd "./fixtures/$1"
+
+# Run the install variation
+case "$2" in
+    cache)
+        bash ../../scripts/variations/cache.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
+    cache+lockfile)
+        bash ../../scripts/variations/cache+lockfile.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
+    cache+node_modules)
+        bash ../../scripts/variations/cache+node_modules.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
+    cache+lockfile+node_modules)
+        bash ../../scripts/variations/cache+lockfile+node_modules.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
+    lockfile)
+        bash ../../scripts/variations/lockfile.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
+    lockfile+node_modules)
+        bash ../../scripts/variations/lockfile+node_modules.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
+    node_modules)
+        bash ../../scripts/variations/node_modules.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
+    clean)
+        bash ../../scripts/variations/clean.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
+    run)
+        bash ../../scripts/variations/run.sh "../../scripts" "../../results" "run" "run"
+        ;;
+    *)
+        echo "Error: Unknown install variation '$2'"
+        exit 1
+        ;;
+esac
+
+popd
+echo "Installation benchmark suite completed successfully!"