Skip to content

Commit 918bd0d

Browse files
adriangbadriangb
committed
add missing file
1 parent 0d299c1 commit 918bd0d

File tree

1 file changed

+240
-0
lines changed

1 file changed

+240
-0
lines changed

‎.github/workflows/pr_benchmarks.yml

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: PR Benchmarks
19+
20+
on:
21+
workflow_dispatch:
22+
inputs:
23+
pr_number:
24+
description: 'PR Number'
25+
required: true
26+
pr_head_sha:
27+
description: 'PR Head SHA'
28+
required: true
29+
base_branch:
30+
description: 'Base branch to compare against (usually main)'
31+
required: true
32+
default: 'main'
33+
base_sha:
34+
description: 'Exact SHA of base branch to compare against'
35+
required: true
36+
benchmarks:
37+
description: 'Space-separated list of benchmarks to run'
38+
required: true
39+
default: 'tpch_mem clickbench_partitioned'
40+
comment_id:
41+
description: 'ID of the comment that triggered the benchmarks'
42+
required: true
43+
44+
permissions:
45+
contents: read
46+
pull-requests: write
47+
48+
env:
49+
RUST_BACKTRACE: 1
50+
CARGO_TERM_COLOR: always
51+
52+
jobs:
53+
benchmark:
54+
name: Run PR Benchmarks
55+
runs-on: ubuntu-latest-large
56+
steps:
57+
- name: Checkout PR branch
58+
uses: actions/checkout@v4
59+
with:
60+
ref: ${{ github.event.inputs.pr_head_sha }}
61+
path: pr_branch
62+
63+
- name: Checkout base branch
64+
uses: actions/checkout@v4
65+
with:
66+
ref: ${{ github.event.inputs.base_sha }}
67+
path: base_branch
68+
69+
- name: Install Python
70+
uses: actions/setup-python@v5
71+
with:
72+
python-version: '3.10'
73+
cache: 'pip'
74+
75+
- name: Setup Rust toolchain
76+
uses: dtolnay/rust-toolchain@stable
77+
with:
78+
toolchain: stable
79+
80+
- name: Generate benchmark data
81+
run: |
82+
# Run data generation for each benchmark
83+
cd pr_branch/benchmarks
84+
85+
# Parse benchmarks from input
86+
IFS=' ' read -r -a BENCHMARKS <<< "${{ github.event.inputs.benchmarks }}"
87+
88+
# Generate data for each benchmark
89+
for benchmark in "${BENCHMARKS[@]}"; do
90+
echo "Generating data for $benchmark..."
91+
./bench.sh data "$benchmark"
92+
done
93+
94+
- name: Run PR branch benchmarks
95+
id: pr_benchmarks
96+
run: |
97+
# Navigate to PR branch
98+
cd pr_branch/benchmarks
99+
100+
# Parse benchmarks from input
101+
IFS=' ' read -r -a BENCHMARKS <<< "${{ github.event.inputs.benchmarks }}"
102+
103+
# Use the branch name as results name
104+
BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)
105+
BRANCH_NAME=${BRANCH_NAME//\//_}
106+
107+
# Run each benchmark
108+
for benchmark in "${BENCHMARKS[@]}"; do
109+
echo "Running $benchmark on PR branch..."
110+
RESULTS_NAME="$BRANCH_NAME" ./bench.sh run "$benchmark"
111+
done
112+
113+
echo "pr_results_dir=pr_branch/benchmarks/results/$BRANCH_NAME" >> $GITHUB_OUTPUT
114+
115+
- name: Run base branch benchmarks
116+
id: base_benchmarks
117+
run: |
118+
# Navigate to base branch
119+
cd base_branch/benchmarks
120+
121+
# Parse benchmarks from input
122+
IFS=' ' read -r -a BENCHMARKS <<< "${{ github.event.inputs.benchmarks }}"
123+
124+
# Use 'base_branch' as results name
125+
BRANCH_NAME="base_${BRANCH_NAME:-main}"
126+
BRANCH_NAME=${BRANCH_NAME//\//_}
127+
128+
# Run each benchmark
129+
for benchmark in "${BENCHMARKS[@]}"; do
130+
echo "Running $benchmark on base branch..."
131+
RESULTS_NAME="$BRANCH_NAME" ./bench.sh run "$benchmark"
132+
done
133+
134+
echo "base_results_dir=base_branch/benchmarks/results/$BRANCH_NAME" >> $GITHUB_OUTPUT
135+
136+
- name: Install comparison requirements
137+
run: |
138+
# Setup virtual environment with requirements
139+
cd pr_branch/benchmarks
140+
pip install -r requirements.txt
141+
142+
- name: Compare benchmark results
143+
id: compare
144+
run: |
145+
# Navigate to PR branch benchmark directory
146+
cd pr_branch/benchmarks
147+
148+
# Parse benchmarks from input
149+
IFS=' ' read -r -a BENCHMARKS <<< "${{ github.event.inputs.benchmarks }}"
150+
151+
# Initialize results variable
152+
COMPARISON_RESULTS=""
153+
154+
# Get the directory names
155+
PR_RESULTS_DIR="${{ steps.pr_benchmarks.outputs.pr_results_dir }}"
156+
BASE_RESULTS_DIR="${{ steps.base_benchmarks.outputs.base_results_dir }}"
157+
158+
# For each benchmark, run comparison
159+
for benchmark in "${BENCHMARKS[@]}"; do
160+
echo "Comparing $benchmark results..."
161+
162+
# Determine result file names based on benchmark
163+
if [[ "$benchmark" == "tpch" ]]; then
164+
RESULT_FILE="tpch_sf1.json"
165+
elif [[ "$benchmark" == "tpch_mem" ]]; then
166+
RESULT_FILE="tpch_mem_sf1.json"
167+
elif [[ "$benchmark" == "tpch10" ]]; then
168+
RESULT_FILE="tpch_sf10.json"
169+
elif [[ "$benchmark" == "tpch_mem10" ]]; then
170+
RESULT_FILE="tpch_mem_sf10.json"
171+
elif [[ "$benchmark" == "clickbench_1" ]]; then
172+
RESULT_FILE="clickbench_1.json"
173+
elif [[ "$benchmark" == "clickbench_partitioned" ]]; then
174+
RESULT_FILE="clickbench_partitioned.json"
175+
elif [[ "$benchmark" == "clickbench_extended" ]]; then
176+
RESULT_FILE="clickbench_extended.json"
177+
elif [[ "$benchmark" == "imdb" ]]; then
178+
RESULT_FILE="imdb.json"
179+
elif [[ "$benchmark" == "external_aggr" ]]; then
180+
RESULT_FILE="external_aggr.json"
181+
elif [[ "$benchmark" == "sort_tpch" ]]; then
182+
RESULT_FILE="sort_tpch.json"
183+
else
184+
RESULT_FILE="$benchmark.json"
185+
fi
186+
187+
# Check if both result files exist
188+
if [[ -f "$PR_RESULTS_DIR/$RESULT_FILE" && -f "$BASE_RESULTS_DIR/$RESULT_FILE" ]]; then
189+
# Run comparison and capture output
190+
OUTPUT=$(python compare.py "$PR_RESULTS_DIR/$RESULT_FILE" "$BASE_RESULTS_DIR/$RESULT_FILE")
191+
COMPARISON_RESULTS+="## $benchmark\n\n\`\`\`\n$OUTPUT\n\`\`\`\n\n"
192+
else
193+
COMPARISON_RESULTS+="## $benchmark\n\nResults not available for comparison.\n\n"
194+
fi
195+
done
196+
197+
# Save comparison results to file for use in PR comment
198+
echo -e "$COMPARISON_RESULTS" > /tmp/benchmark_comparison.txt
199+
200+
- name: Post results as PR comment
201+
uses: actions/github-script@v7
202+
with:
203+
github-token: ${{ secrets.GITHUB_TOKEN }}
204+
script: |
205+
const fs = require('fs');
206+
const pr_number = ${{ github.event.inputs.pr_number }};
207+
const pr_head_sha = '${{ github.event.inputs.pr_head_sha }}';
208+
const base_branch = '${{ github.event.inputs.base_branch }}';
209+
const base_sha = '${{ github.event.inputs.base_sha }}';
210+
const comment_id = ${{ github.event.inputs.comment_id }};
211+
212+
// Read comparison results
213+
const comparisonText = fs.readFileSync('/tmp/benchmark_comparison.txt', 'utf8');
214+
215+
// Parse benchmarks from input
216+
const benchmarks = '${{ github.event.inputs.benchmarks }}'.split(' ');
217+
218+
// Create comment with results in collapsible sections
219+
const comment = `## 📊 Benchmark Results
220+
221+
<details>
222+
<summary>Expand for detailed results</summary>
223+
224+
\${comparisonText}
225+
</details>
226+
227+
Benchmarks run: \${benchmarks.join(', ')}
228+
229+
Comparing PR branch (\`\${pr_head_sha.substring(0, 8)}\`) with base branch \`\${base_branch}\` (\`\${base_sha.substring(0, 8)}\`)
230+
231+
Triggered by [this comment](https://github.com/\${context.repo.owner}/\${context.repo.repo}/pull/\${pr_number}#issuecomment-\${comment_id})
232+
`;
233+
234+
// Post comment to PR
235+
await github.rest.issues.createComment({
236+
owner: context.repo.owner,
237+
repo: context.repo.repo,
238+
issue_number: pr_number,
239+
body: comment
240+
});

0 commit comments

Comments
 (0)