From faa4d583101db64847f037bf8a2e56c3df3b05a1 Mon Sep 17 00:00:00 2001 From: Paschalis Mpeis Date: Sat, 28 Jun 2025 10:01:54 +0100 Subject: [PATCH 1/5] [BOLT] Add sanity check for frozen llvm-bolt Some patches can cause the llvm-bolt binary to hang, which stalls or fails the test pipeline. Add a simple sanity check that runs: ``` llvm-bolt --version ``` with a 30-second timeout. If the command does not complete in time, flunk the build. Set maxTime for nfc-check-validation and reduce the number of lit workers for in-tree tests. --- zorg/buildbot/builders/BOLTBuilder.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/zorg/buildbot/builders/BOLTBuilder.py b/zorg/buildbot/builders/BOLTBuilder.py index a79afbf8..26901525 100644 --- a/zorg/buildbot/builders/BOLTBuilder.py +++ b/zorg/buildbot/builders/BOLTBuilder.py @@ -130,6 +130,16 @@ def getBOLTCmakeBuildFactory( haltOnFailure=False, flunkOnFailure=False, env=env), + ShellCommand( + name='llvm-bolt-version-check', + command=(f"{boltNew} --version"), + description=('Check that llvm-bolt binary passes a simple test' + 'before proceeding with testing.'), + descriptionDone=["llvm-bolt --version"], + haltOnFailure=True, + flunkOnFailure=True, + maxTime=30, + env=env), # Validate that NFC-mode comparison is meaningful by checking: # - the old and new binaries exist # - no unique IDs are embedded in the binaries @@ -154,6 +164,7 @@ def getBOLTCmakeBuildFactory( haltOnFailure=False, warnOnFailure=True, warnOnWarnings=True, + maxTime=20, decodeRC={0: SUCCESS, 1: FAILURE, 2: WARNINGS}, descriptionDone=["NFC-Mode Validation"], env=env), @@ -177,7 +188,7 @@ def getBOLTCmakeBuildFactory( # relevant source code changes are detected. LitTestCommand( name='nfc-check-bolt', - command=["ninja", "check-bolt"], + command=("LIT_OPTS='-j2' ninja check-bolt"), description=["running", "NFC", "check-bolt"], descriptionDone=["NFC", "check-bolt", "completed"], warnOnFailure=True, From 352259dacacb809ff6d56a90c82be72ef1dbc8bf Mon Sep 17 00:00:00 2001 From: Paschalis Mpeis Date: Sat, 28 Jun 2025 13:40:08 +0100 Subject: [PATCH 2/5] Use nice when running tests --- zorg/buildbot/builders/BOLTBuilder.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zorg/buildbot/builders/BOLTBuilder.py b/zorg/buildbot/builders/BOLTBuilder.py index 26901525..fe1763b2 100644 --- a/zorg/buildbot/builders/BOLTBuilder.py +++ b/zorg/buildbot/builders/BOLTBuilder.py @@ -188,7 +188,7 @@ def getBOLTCmakeBuildFactory( # relevant source code changes are detected. LitTestCommand( name='nfc-check-bolt', - command=("LIT_OPTS='-j2' ninja check-bolt"), + command=("LIT_OPTS='-j2' nice -n 5 ninja check-bolt"), description=["running", "NFC", "check-bolt"], descriptionDone=["NFC", "check-bolt", "completed"], warnOnFailure=True, @@ -199,8 +199,7 @@ def getBOLTCmakeBuildFactory( # Run out-of-tree large tests if the llvm-bolt binary has changed. LitTestCommand( name='nfc-check-large-bolt', - command=['bin/llvm-lit', '-sv', '-j2', - 'tools/bolttests'], + command=('nice -n 5 bin/llvm-lit -sv -j2 tools/bolttests'), description=["running", "NFC", "check-large-bolt"], descriptionDone=["NFC", "check-large-bolt", "completed"], warnOnFailure=True, From 9f1fea273f3b8fcd557153e185b6ae0c03662d00 Mon Sep 17 00:00:00 2001 From: Paschalis Mpeis Date: Mon, 30 Jun 2025 16:32:14 +0100 Subject: [PATCH 3/5] Addressing reviewers --- zorg/buildbot/builders/BOLTBuilder.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/zorg/buildbot/builders/BOLTBuilder.py b/zorg/buildbot/builders/BOLTBuilder.py index fe1763b2..2f28898e 100644 --- a/zorg/buildbot/builders/BOLTBuilder.py +++ b/zorg/buildbot/builders/BOLTBuilder.py @@ -130,6 +130,8 @@ def getBOLTCmakeBuildFactory( haltOnFailure=False, flunkOnFailure=False, env=env), + # Verify that the llvm-bolt binary can report its version within a + # reasonable amount of time. ShellCommand( name='llvm-bolt-version-check', command=(f"{boltNew} --version"), @@ -164,7 +166,7 @@ def getBOLTCmakeBuildFactory( haltOnFailure=False, warnOnFailure=True, warnOnWarnings=True, - maxTime=20, + maxTime=30, decodeRC={0: SUCCESS, 1: FAILURE, 2: WARNINGS}, descriptionDone=["NFC-Mode Validation"], env=env), @@ -185,10 +187,12 @@ def getBOLTCmakeBuildFactory( haltOnFailure=False, env=env), # Run in-tree tests if the llvm-bolt binary has changed, or if - # relevant source code changes are detected. + # relevant source code changes are detected. Lower scheduling + # priority with nice to reduce CPU contention in virtualized + # environments. LitTestCommand( name='nfc-check-bolt', - command=("LIT_OPTS='-j2' nice -n 5 ninja check-bolt"), + command=("nice -n 5 ninja check-bolt"), description=["running", "NFC", "check-bolt"], descriptionDone=["NFC", "check-bolt", "completed"], warnOnFailure=True, @@ -197,6 +201,7 @@ def getBOLTCmakeBuildFactory( doStepIf=FileDoesNotExist(f"build/{skipInTree}"), env=env), # Run out-of-tree large tests if the llvm-bolt binary has changed. + # Lower scheduling priority, as above. LitTestCommand( name='nfc-check-large-bolt', command=('nice -n 5 bin/llvm-lit -sv -j2 tools/bolttests'), From 8e2ded2b3215e746f5e09d1bbd28a77635c99aa9 Mon Sep 17 00:00:00 2001 From: Paschalis Mpeis Date: Wed, 2 Jul 2025 17:21:23 +0100 Subject: [PATCH 4/5] Add more comments in code. --- zorg/buildbot/builders/BOLTBuilder.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zorg/buildbot/builders/BOLTBuilder.py b/zorg/buildbot/builders/BOLTBuilder.py index 2f28898e..ebbdb2d6 100644 --- a/zorg/buildbot/builders/BOLTBuilder.py +++ b/zorg/buildbot/builders/BOLTBuilder.py @@ -103,7 +103,7 @@ def getBOLTCmakeBuildFactory( boltOld = "bin/llvm-bolt.old" f.addSteps([ - # Cleanup binaries and markers from previous NFC-mode runs. + # Cleanup old/new binaries and markers from previous NFC-mode runs. ShellCommand( name='clean-nfc-check', command=( @@ -114,7 +114,9 @@ def getBOLTCmakeBuildFactory( haltOnFailure=False, flunkOnFailure=False, env=env), - # Build the current and previous revision of llvm-bolt. + # Build the current and previous revision of llvm-bolt as + # llvm-bolt.new and llvm-bolt.old. Also, creates a marker to force + # in-tree tests in case additional source code changes are detected. ShellCommand( name='nfc-check-setup', command=[ @@ -189,7 +191,7 @@ def getBOLTCmakeBuildFactory( # Run in-tree tests if the llvm-bolt binary has changed, or if # relevant source code changes are detected. Lower scheduling # priority with nice to reduce CPU contention in virtualized - # environments. + # environments. This step relinks the llvm-bolt binary if needed. LitTestCommand( name='nfc-check-bolt', command=("nice -n 5 ninja check-bolt"), From 114db12b7c8adde9b914dc71ab87e2328fa57772 Mon Sep 17 00:00:00 2001 From: Paschalis Mpeis Date: Wed, 2 Jul 2025 17:25:10 +0100 Subject: [PATCH 5/5] Drop -s from llvm-lit This would produce a list of the ran lit tests on the output --- zorg/buildbot/builders/BOLTBuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zorg/buildbot/builders/BOLTBuilder.py b/zorg/buildbot/builders/BOLTBuilder.py index ebbdb2d6..a8f1cd68 100644 --- a/zorg/buildbot/builders/BOLTBuilder.py +++ b/zorg/buildbot/builders/BOLTBuilder.py @@ -206,7 +206,7 @@ def getBOLTCmakeBuildFactory( # Lower scheduling priority, as above. LitTestCommand( name='nfc-check-large-bolt', - command=('nice -n 5 bin/llvm-lit -sv -j2 tools/bolttests'), + command=('nice -n 5 bin/llvm-lit -v -j2 tools/bolttests'), description=["running", "NFC", "check-large-bolt"], descriptionDone=["NFC", "check-large-bolt", "completed"], warnOnFailure=True,