Skip to content

Commit dd2524f

Browse files
Michael137Michael137
authored
[zorg][lldb] Add new LLDB metrics bot (#278)
This patch adds a new job to collect LLDB metrics. This is heavily based on the `debuginfo-statistics` job (but currently doesn't publish data to LNT). Currently this job would do the following: 1. Check out the Clang 19.x release and build it 2. Use the LLDB and Clang from the `lldb-cmake-intel` job (not actually sure if that job publishes the right artifacts at the moment) to run the `run_lldb_metrics.sh` script. 3. Said script will attach LLDB to Clang/LLDB and run various commands. Then it will dump the `statistics dump` command to `stdout` (note we don't do any kind of averaging of these over multiple runs, since the metrics we care about **should** stable across runs). We also currently run these test-scenarios through `hyperfine` and dump the timing data. But maybe for a first attempt this isn't necessary.
1 parent 699c3ba commit dd2524f

File tree

2 files changed

+388
-0
lines changed

2 files changed

+388
-0
lines changed

‎zorg/jenkins/jobs/jobs/lldb-statistics

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
#!/usr/bin/env groovy
2+
3+
// This pipeline collects LLDB metrics by attaching LLDB to stable a
4+
// version of clang-19/lldb-19, run various LLDB commands and then
5+
// dump LLDB statistics.
6+
7+
// These are the LLVM build directories involved:
8+
//
9+
// (a) The "host" compiler/LLDB is taken from whatever the LLDB incremental built/used.
10+
// The metrics we collect are from that "host" LLDB that we fetched. Throughout this
11+
// file it is commonly referred to as 'host-compiler' or 'HOST_BUILD_DIR'.
12+
//
13+
// (b) The debugger/compiler that we're debugging as part of metrics collection is pinned to
14+
// the llvm-19.x release and is supposed to be unchanging across runs of this job.
15+
// Throughout this file we refer to it as "historic" or 'clang-19'. We build this
16+
// historic compiler with a 2-stage bootstrap build (the first stage in Release to
17+
// speed up the build process, and the second stage in Debug since we need to attach
18+
// LLDB to it).
19+
//
20+
// (c) We use the compiler from (a) to build the "historic" Clang/LLDB.
21+
//
22+
// (d) The compiler compiling the debugger in (a) is the Clang produced by the clang-stage2
23+
// buildbot and happens outside the purview job.
24+
//
25+
// In summary, the only stable version of Clang/LLVM is the one we use as the inferior during
26+
// metrics collection.
27+
28+
pipeline {
29+
options {
30+
disableConcurrentBuilds()
31+
32+
timeout(time: 12, unit: 'HOURS')
33+
}
34+
35+
parameters {
36+
string(name: 'LABEL', defaultValue: params.LABEL ?: 'macos-x86_64', description: 'Node label to run on')
37+
string(name: 'GIT_SHA', defaultValue: params.GIT_REVISION ?: '*/release/19.x', description: 'Git commit to build.')
38+
string(name: 'ARTIFACT', defaultValue: params.ARTIFACT ?: 'lldb-cmake-intel/latest', description: 'Clang/LLDB artifact to use')
39+
booleanParam(name: 'CLEAN', defaultValue: params.CLEAN ?: false, description: 'Wipe the build directory?')
40+
}
41+
42+
agent {
43+
node {
44+
label params.LABEL
45+
}
46+
}
47+
stages {
48+
stage('Checkout') {
49+
steps {
50+
script {
51+
if(params.CLEAN) {
52+
deleteDir()
53+
}
54+
}
55+
56+
timeout(30) {
57+
dir('src/clang-19') {
58+
checkout([$class: 'GitSCM', branches: [
59+
[name: params.GIT_SHA]
60+
], userRemoteConfigs: [
61+
[url: 'https://github.com/llvm/llvm-project.git']
62+
], extensions: [
63+
[$class: 'CloneOption',
64+
noTags: true, timeout: 30]
65+
]])
66+
}
67+
dir('llvm-zorg') {
68+
checkout([$class: 'GitSCM', branches: [
69+
[name: '*/main']
70+
], userRemoteConfigs: [
71+
[url: 'https://github.com/llvm/llvm-zorg.git']
72+
]])
73+
}
74+
}
75+
}
76+
}
77+
stage('Setup Venv') {
78+
environment {
79+
PATH="$PATH:/usr/bin:/usr/local/bin"
80+
}
81+
steps {
82+
sh '''
83+
rm -rf venv
84+
python3 -m venv venv
85+
set +u
86+
source ./venv/bin/activate
87+
pip install -r ./llvm-zorg/zorg/jenkins/jobs/requirements.txt
88+
set -u
89+
'''
90+
}
91+
}
92+
stage('Fetch Artifact') {
93+
environment {
94+
PATH="$PATH:/usr/bin:/usr/local/bin"
95+
}
96+
steps {
97+
withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) {
98+
sh """
99+
source ./venv/bin/activate
100+
echo "ARTIFACT=${params.ARTIFACT}"
101+
python llvm-zorg/zorg/jenkins/monorepo_build.py fetch
102+
ls $WORKSPACE/host-compiler/lib/clang/
103+
VERSION=`ls $WORKSPACE/host-compiler/lib/clang/`
104+
"""
105+
}
106+
}
107+
}
108+
stage('Build') {
109+
environment {
110+
PATH="$PATH:/usr/bin:/usr/local/bin"
111+
SRC="$WORKSPACE/src"
112+
BUILD="$WORKSPACE/clang-19-build"
113+
TEST="$WORKSPACE/test"
114+
RESULTS="$WORKSPACE/results"
115+
CC="$WORKSPACE/host-compiler/bin/clang"
116+
CXX="$WORKSPACE/host-compiler/bin/clang++"
117+
HISTORIC_COMPILER="clang-19"
118+
}
119+
steps {
120+
withCredentials([string(credentialsId: 's3_resource_bucket', variable: 'S3_BUCKET')]) {
121+
sh '''
122+
source ./venv/bin/activate
123+
124+
cd src/clang-19
125+
git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true
126+
127+
git_desc=$(git describe --match "first_commit")
128+
export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-")
129+
130+
sha=$(echo ${git_desc} | cut -f 3 -d "-")
131+
export GIT_SHA=${sha:1}
132+
133+
cd -
134+
135+
set -eux
136+
137+
$CXX --version
138+
LLVM_REV=${GIT_DISTANCE}
139+
140+
mkdir -p $HISTORIC_COMPILER-src
141+
mkdir -p $HISTORIC_COMPILER-build
142+
rsync -a $SRC/$HISTORIC_COMPILER/ $HISTORIC_COMPILER-src/
143+
cd $HISTORIC_COMPILER-build
144+
cmake ../$HISTORIC_COMPILER-src/llvm \
145+
-DCMAKE_BUILD_TYPE=Release \
146+
-DLLVM_ENABLE_PROJECTS="clang;lldb" \
147+
-DLLVM_ENABLE_ASSERTIONS=Off \
148+
-DLLVM_ENABLE_MODULES=Off \
149+
-DLLDB_INCLUDE_TESTS=Off \
150+
-DLLDB_ENABLE_PYTHON=Off \
151+
-DLLDB_ENABLE_LUA=Off \
152+
-DLLVM_TARGETS_TO_BUILD='X86;AArch64' \
153+
-DCMAKE_EXPORT_COMPILE_COMMANDS=On \
154+
-DCMAKE_C_COMPILER=$CC \
155+
-DCMAKE_CXX_COMPILER=$CXX \
156+
-DCLANG_ENABLE_BOOTSTRAP=On \
157+
-DCLANG_BOOTSTRAP_PASSTHROUGH="LLDB_INCLUDE_TESTS;LLDB_ENABLE_PYTHON;LLDB_ENABLE_LUA" \
158+
-DBOOTSTRAP_CMAKE_BUILD_TYPE=Debug \
159+
-G Ninja
160+
cmake --build .
161+
cd ../..
162+
'''
163+
}
164+
}
165+
}
166+
stage('Run metrics') {
167+
environment {
168+
HOST_BUILD_DIR="$WORKSPACE/host-compiler"
169+
HISTORIC_BUILD_DIR="$WORKSPACE/clang-19-build/tools/clang/stage2-bins"
170+
}
171+
steps {
172+
sh '''
173+
./llvm-zorg/zorg/jenkins/jobs/util/run_lldb_metrics.sh $HOST_BUILD_DIR $HISTORIC_BUILD_DIR
174+
'''
175+
}
176+
}
177+
178+
// TODO: for now we just dump the statistics to the console.
179+
//stage('Submit debuginfo statistics to LNT') {
180+
// steps {
181+
// sh '''
182+
// source ./venv/bin/activate
183+
184+
// cd src/clang-13
185+
// git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true
186+
187+
// git_desc=$(git describe --match "first_commit")
188+
// export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-")
189+
190+
// cd -
191+
192+
// python llvm-zorg/zorg/jenkins/jobs/util/submit-debuginfo-statistics-to-lnt.py
193+
// '''
194+
// }
195+
//}
196+
}
197+
}

‎zorg/jenkins/jobs/util/run_lldb_metrics.sh

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
#!/bin/bash
2+
3+
# Invoke as: ./run_lldb_metrics.sh /path/to/llvm/under/test /path/to/debug/llvm/build
4+
5+
set -xeou pipefail
6+
7+
# This directory contains the LLDB under test for which we collect metrics.
8+
LLVM_BUILD_DIR=$1
9+
if [ -z "${LLVM_BUILD_DIR}" ] || [ ! -d "${LLVM_BUILD_DIR}" ]; then
10+
echo "Invalid path to host Clang specified: ${LLVM_BUILD_DIR}"
11+
exit 1
12+
fi
13+
14+
# This directory contains a debug build of clang and LLDB. We attach
15+
# the LLDB under test (in LLVM_BUILD_DIR) to these debug binaries
16+
# to collect metrics.
17+
DEBUG_BUILD_DIR=$2
18+
if [ -z "${DEBUG_BUILD_DIR}" ] || [ ! -d "${DEBUG_BUILD_DIR}" ]; then
19+
echo "Invalid path to host debug Clang/LLDB specified: ${DEBUG_BUILD_DIR}"
20+
exit 1
21+
fi
22+
23+
ARTIFACTS=/tmp/lldb-metrics
24+
RESULTS_DIR=${ARTIFACTS}/results
25+
HYPERFINE_RUNS=5
26+
HYPERFINE_WARMUP=3
27+
SYSROOT=`xcrun --show-sdk-path`
28+
TEST_COMPILE_FLAGS="-g -O0 -isysroot ${SYSROOT}"
29+
TEST_SRC_FILE=${ARTIFACTS}/main.cpp
30+
31+
profile_clang() {
32+
local test_case_name="clang_$1"
33+
34+
# Invocation to be profiled.
35+
local profile_invocation="$2"
36+
37+
# Debuggee that LLDB will launch.
38+
local debuggee_invocation="${DEBUG_BUILD_DIR}/bin/clang++ -c ${TEST_COMPILE_FLAGS} ${TEST_SRC_FILE} -o $ARTIFACTS/a.o"
39+
40+
# Where to write the 'statistics dump' output to.
41+
local stats_filename="${RESULTS_DIR}/${test_case_name}.json"
42+
43+
# Run test-case and collect statistics.
44+
eval "${profile_invocation} -o 'script -- f=open(\"${stats_filename}\",\"w\"); lldb.debugger.SetOutputFileHandle(f,True); lldb.debugger.HandleCommand(\"statistics dump\")' --batch -- ${debuggee_invocation}"
45+
46+
[[ -f $stats_filename ]] && cat $stats_filename
47+
}
48+
49+
profile_lldb() {
50+
local test_case_name="lldb_$1"
51+
52+
# Invocation to be profiled.
53+
local profile_invocation="$2"
54+
55+
local debuggee="${ARTIFACTS}/a.out"
56+
${DEBUG_BUILD_DIR}/bin/clang++ -isysroot ${SYSROOT} ${TEST_COMPILE_FLAGS} ${TEST_SRC_FILE} -o ${debuggee}
57+
58+
# Debuggee that LLDB will launch.
59+
local debuggee_invocation="${DEBUG_BUILD_DIR}/bin/lldb ${debuggee} -o 'br se -p return' -o run -o 'expr fib(1)'"
60+
61+
# Where to write the 'statistics dump' output to.
62+
local stats_filename="${RESULTS_DIR}/${test_case_name}.json"
63+
64+
# Run test-case and collect statistics.
65+
eval "${profile_invocation} -o 'script -- f=open(\"${stats_filename}\",\"w\"); lldb.debugger.SetOutputFileHandle(f,True); lldb.debugger.HandleCommand(\"statistics dump\")' --batch -- ${debuggee_invocation}"
66+
67+
[[ -f $stats_filename ]] && cat $stats_filename
68+
}
69+
70+
# Clean previous results
71+
72+
rm -rf $ARTIFACTS
73+
mkdir $ARTIFACTS
74+
mkdir $RESULTS_DIR
75+
76+
cat >${TEST_SRC_FILE} <<EOL
77+
#include <map>
78+
79+
int fib(int n) {
80+
static auto cache = [] {
81+
auto ans = std::map<int, int>();
82+
ans[0] = 0;
83+
ans[1] = 1;
84+
return ans;
85+
}();
86+
if (auto it = cache.find(n); it != cache.end()) return it->second;
87+
auto ans = fib(n - 1) + fib(n - 2);
88+
cache[ans] = ans;
89+
return ans;
90+
}
91+
92+
int main() {
93+
fib(5);
94+
return 0;
95+
}
96+
EOL
97+
98+
# Benchmarks
99+
# ==========
100+
#
101+
# Attaches the LLDB under test to a debug build of Clang/LLDB. We stop in a member
102+
# function of a "large" class (in this case clang::CodeGen::CodeGenFunction or
103+
# lldb_private::ClangASTSource). We then run various LLDB commands and collect a
104+
# `statistics dump` afterwards. Currently the scenarios are:
105+
#
106+
# * frame_status: this just runs up to (and including) stopping in
107+
# a function, which then triggers formatting of the frame
108+
# status (which can be non-trivial since we trigger
109+
# data-formatters for function arguments and completion of
110+
# argument variables).
111+
#
112+
# * expr_deref: dereferences the "llvm::Function *Fn" function argument
113+
# through the expression evaluator. This will trigger completion
114+
# of the CodeGenFunction context and the llvm::Function class.
115+
#
116+
# * expr_method_call: call a method on Fn->isVarArg(). Similar to "expr_deref"
117+
# just with an additional function call.
118+
#
119+
# * expr_re_eval: dereference "llvm::Function *Fn" multiple times consecutively,
120+
# in the hopes that some of the work doesn't have to be re-done.
121+
#
122+
# * expr_two_stops: We run the "expr_deref", and then continue to another breakpoint
123+
# inside LLVM and run another set of expressions, testing the expression
124+
# evaluator's behaviour when stopping in different LLDB modules.
125+
#
126+
# * var_then_expr: Run "frame var" followed by expression evaluation.
127+
#
128+
# * var: Run "frame var", which triggers data-formatters and completion of local
129+
# variables.
130+
131+
# Clang benchmarks
132+
# ================
133+
134+
profile_clang \
135+
"frame_status" \
136+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b CodeGenFunction::GenerateCode' -o run"
137+
138+
profile_clang \
139+
"expr_deref" \
140+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b CodeGenFunction::GenerateCode' -o run -o 'expr *Fn'"
141+
142+
profile_clang \
143+
"expr_method_call" \
144+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b CodeGenFunction::GenerateCode' -o run -o 'expr Fn->isVarArg()'"
145+
146+
profile_clang \
147+
"expr_re_eval" \
148+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b CodeGenFunction::GenerateCode' -o run -o 'expr *Fn' -o 'expr *Fn'"
149+
150+
profile_clang \
151+
"expr_two_stops" \
152+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b CodeGenFunction::GenerateCode' -o run -o 'expr *Fn' -o 'br del 1' -o 'b AsmPrinter::emitFunctionBody' -o c -o 'expr this->isVerbose()' -o 'expr TM'"
153+
154+
profile_clang \
155+
"var_then_expr" \
156+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b CodeGenFunction::GenerateCode' -o run -o up -o 'frame var' -o down -o 'expr *this'"
157+
158+
profile_clang \
159+
"var" \
160+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b CodeGenFunction::GenerateCode' -o run -o 'frame var'"
161+
162+
# LLDB benchmarks
163+
# ===============
164+
165+
profile_lldb \
166+
"frame_status" \
167+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b ClangASTSource::FindExternalVisibleDeclsByName' -o run"
168+
169+
profile_lldb \
170+
"expr_deref" \
171+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b ClangASTSource::FindExternalVisibleDeclsByName' -o run -o 'expr m_active_lexical_decls'"
172+
173+
profile_lldb \
174+
"expr_method_call" \
175+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b ClangASTSource::FindExternalVisibleDeclsByName' -o run -o 'expr clang_decl_name.getAsString()'"
176+
177+
profile_lldb \
178+
"expr_re_eval" \
179+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b ClangASTSource::FindExternalVisibleDeclsByName' -o run -o 'expr m_active_lexical_decls' -o 'expr m_active_lexical_decls'"
180+
181+
profile_lldb \
182+
"expr_two_stops" \
183+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b ClangASTSource::FindExternalVisibleDeclsByName' -o run -o 'expr m_active_lexical_decls' -o 'b CodeGeneratorImpl::HandleTranslationUnit' -o 'br del 1' -o c -o 'expr Builder' -o 'expr Ctx.getLangOpts()'"
184+
185+
profile_lldb \
186+
"var_then_expr" \
187+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b ClangASTSource::FindExternalVisibleDeclsByName' -o run -o up -o 'frame var' -o down -o 'expr *this'"
188+
189+
profile_lldb \
190+
"var" \
191+
"${LLVM_BUILD_DIR}/bin/lldb -o 'b ClangASTSource::FindExternalVisibleDeclsByName' -o run -o 'frame var'"

0 commit comments

Comments
 (0)