tools: add benchmark runner to test-shared
#4
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark and shared librairies | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - .mailmap | |
| - '**.md' | |
| - AUTHORS | |
| - doc/** | |
| - .github/** | |
| - '!.github/workflows/test-shared.yml' | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| push: | |
| branches: | |
| - main | |
| - canary | |
| - v[0-9]+.x-staging | |
| - v[0-9]+.x | |
| paths-ignore: | |
| - .mailmap | |
| - '**.md' | |
| - AUTHORS | |
| - doc/** | |
| - .github/** | |
| - '!.github/workflows/test-shared.yml' | |
| workflow_dispatch: | |
| inputs: | |
| repo: | |
| type: string | |
| description: GitHub repository to fetch from (default to the current repo) | |
| pr_id: | |
| type: number | |
| required: true | |
| description: The PR to test | |
| commit: | |
| required: true | |
| type: string | |
| description: The expect HEAD of the PR | |
| category: | |
| required: true | |
| type: string | |
| description: The category (or categories) of tests to run, for example buffers, cluster etc. Maps to a folders in node/benchmark | |
| filter: | |
| type: string | |
| description: A substring to restrict the benchmarks to run in a category. e.g. `net-c2c` | |
| runs: | |
| type: number | |
| default: 30 | |
| description: How many times to repeat each benchmark | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| env: | |
| FLAKY_TESTS: keep_retrying | |
| permissions: | |
| contents: read | |
| jobs: | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: ubuntu-24.04 | |
| system: x86_64-linux | |
| - runner: ubuntu-24.04-arm | |
| system: aarch64-linux | |
| - runner: macos-13 | |
| system: x86_64-darwin | |
| - runner: macos-latest | |
| system: aarch64-darwin | |
| name: '${{ matrix.system }}: with shared libraries' | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| if: ${{ github.event_name != 'workflow_dispatch' }} | |
| with: | |
| persist-credentials: false | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| with: | |
| repository: ${{ inputs.repo || github.repository }} | |
| ref: refs/pull/${{ inputs.pr_id }}/merge | |
| persist-credentials: false | |
| fetch-depth: 2 | |
| - name: Validate PR head and roll back to base commit | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| [ "$(git rev-parse HEAD^2)" = "$EXPECTED_SHA" ] | |
| git reset HEAD^ --hard | |
| env: | |
| EXPECTED_SHA: ${{ inputs.commit }} | |
| - uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1 | |
| with: | |
| extra_nix_config: sandbox = true | |
| - name: Configure sccache | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| core.exportVariable('SCCACHE_GHA_VERSION', 'on'); | |
| core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on'); | |
| core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); | |
| core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); | |
| - name: Build Node.js ${{ github.event_name == 'workflow_dispatch' && 'on the base commit' || 'and run tests' }} | |
| run: | | |
| nix-shell \ | |
| -I nixpkgs=./tools/nix/pkgs.nix \ | |
| --pure --keep FLAKY_TESTS \ | |
| --keep SCCACHE_GHA_VERSION --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \ | |
| --arg loadJSBuiltinsDynamically false \ | |
| --arg ccache '(import <nixpkgs> {}).sccache' \ | |
| --arg devTools '[]' \ | |
| --arg benchmarkTools '[]' \ | |
| --run ' | |
| make ${{ github.event_name == 'workflow_dispatch' && 'build' || 'run' }}-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS" | |
| ' | |
| - name: Re-build Node.js on the merge commit | |
| # ccache is disabled here to avoid polluting the cache. Local build outputs should make this build relatively quick anyway. | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| mv out/Release/node base_node | |
| git reset FETCH_HEAD --hard | |
| nix-shell \ | |
| -I nixpkgs=./tools/nix/pkgs.nix \ | |
| --pure \ | |
| --arg loadJSBuiltinsDynamically false \ | |
| --arg ccache 'null' \ | |
| --arg devTools '[]' \ | |
| --arg benchmarkTools '[]' \ | |
| --run ' | |
| make -j4 V=1 | |
| ' | |
| - name: Run benchmark | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| nix-shell \ | |
| -I nixpkgs=./tools/nix/pkgs.nix \ | |
| --pure --keep FILTER --keep LC_ALL --keep LANG \ | |
| --arg loadJSBuiltinsDynamically false \ | |
| --arg ccache 'null' \ | |
| --arg icu 'null' \ | |
| --arg sharedLibDeps '{}' \ | |
| --arg devTools '[]' \ | |
| --run ' | |
| set -o pipefail | |
| ./base_node benchmark/compare.js \ | |
| --filter "$FILTER" \ | |
| --runs ${{ inputs.runs }} \ | |
| --old ./base_node --new ./node \ | |
| -- ${{ inputs.category }} \ | |
| | tee /dev/stderr \ | |
| > ${{ matrix.system }}.csv | |
| echo "Benchmark results:" | |
| echo | |
| echo '"'"'```'"'"' | |
| Rscript benchmark/compare.R < ${{ matrix.system }}.csv | |
| echo '"'"'```'"'"' | |
| ' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY" | |
| env: | |
| FILTER: ${{ inputs.filter }} | |
| - name: Upload raw benchmark results | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: csv-${{ matrix.system }} | |
| path: ${{ matrix.system }}.csv | |
| aggregate-benchmark-results: | |
| needs: build | |
| name: Aggregate benchmark results | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout: | | |
| benchmark/*.R | |
| tools/nix/*.nix | |
| *.nix | |
| sparse-checkout-cone-mode: false | |
| - name: Download benchmark raw results | |
| uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 | |
| with: | |
| pattern: csv-* | |
| merge-multiple: true | |
| path: raw-results | |
| - uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1 | |
| with: | |
| extra_nix_config: sandbox = true | |
| - name: Benchmark results | |
| run: | | |
| nix-shell \ | |
| -I nixpkgs=./tools/nix/pkgs.nix \ | |
| --pure --keep LC_ALL --keep LANG \ | |
| --arg loadJSBuiltinsDynamically false \ | |
| --arg ccache 'null' \ | |
| --arg icu 'null' \ | |
| --arg sharedLibDeps '{}' \ | |
| --arg devTools '[]' \ | |
| --run ' | |
| echo "Benchmark results:" | |
| echo | |
| echo '"'"'```'"'"' | |
| awk "FNR==1 && NR!=1{next;}{print}" raw-results/*.csv | Rscript benchmark/compare.R | |
| echo '"'"'```'"'"' | |
| ' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY" |