From 552d29cb757c2f24dc31a2edb5caada8249cf5b0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 10 Jun 2025 11:48:42 -0700 Subject: [PATCH 01/61] naive start --- lld/wasm/SyntheticSections.h | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/lld/wasm/SyntheticSections.h b/lld/wasm/SyntheticSections.h index 068fbed11f4a7..2c391d2815914 100644 --- a/lld/wasm/SyntheticSections.h +++ b/lld/wasm/SyntheticSections.h @@ -467,6 +467,43 @@ class BuildIdSection : public SyntheticSection { uint8_t *hashPlaceholderPtr = nullptr; }; +class BranchHintSection : public SyntheticSection { +public: + BranchHintSection(ArrayRef segments) + : SyntheticSection(llvm::wasm::WASM_SEC_CUSTOM, "metadata.code.branch_hint"), + segments(segments) {} + bool isNeeded() const override { + if (ctx.arg.stripAll && !ctx.arg.keepSections.count(name)) + return false; + return numHints() > 0; + } + void writeBody() override; + size_t numHints() const { + size_t total = 0; + for (auto& funcHints : funcHintsVec) { + total += funcHints.hints.size(); + } + return total; + } + +protected: + // A hint for an instruction. + struct Hint { + // TODO expr loc + bool likely; + }; + + // All the hints in a function. + struct FuncHints { + StringRef funcName; + std::vector hints; + }; + + std::vector funcHintsVec; + + ArrayRef segments; +}; + // Linker generated output sections struct OutStruct { DylinkSection *dylinkSec; @@ -486,6 +523,7 @@ struct OutStruct { ProducersSection *producersSec; TargetFeaturesSection *targetFeaturesSec; BuildIdSection *buildIdSec; + BranchHintSection *branchHintSec; }; extern OutStruct out; From 7294127d65a438d391f7736ed40ede8f9d78dcf1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 10 Jun 2025 12:42:48 -0700 Subject: [PATCH 02/61] more --- lld/wasm/SyntheticSections.cpp | 5 +++++ lld/wasm/Writer.cpp | 2 ++ 2 files changed, 7 insertions(+) diff --git a/lld/wasm/SyntheticSections.cpp b/lld/wasm/SyntheticSections.cpp index 76719596c62e0..5a8151d089ad1 100644 --- a/lld/wasm/SyntheticSections.cpp +++ b/lld/wasm/SyntheticSections.cpp @@ -960,4 +960,9 @@ void BuildIdSection::writeBuildId(llvm::ArrayRef buf) { memcpy(hashPlaceholderPtr, buf.data(), hashSize); } +void BranchHintSection::writeBody() { + for (const InputFunction *f : out.functionSec->inputFunctions) { + } +} + } // namespace wasm::lld diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp index b704677d36c93..24b6cbcca679e 100644 --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -558,6 +558,7 @@ void Writer::addSections() { addSection(out.producersSec); addSection(out.targetFeaturesSec); addSection(out.buildIdSec); + addSection(out.branchHintSec); } void Writer::finalizeSections() { @@ -1702,6 +1703,7 @@ void Writer::createSyntheticSectionsPostLayout() { out.dataCountSec = make(segments); out.linkingSec = make(initFunctions, segments); out.nameSec = make(segments); + out.branchHintSec = make(segments); } void Writer::run() { From 80210db070d1fd8550f568401d02b46cac45fc95 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 08:11:21 -0700 Subject: [PATCH 03/61] bad --- lld/wasm/InputChunks.h | 2 +- lld/wasm/SyntheticSections.h | 2 +- llvm/include/llvm/ObjectYAML/WasmYAML.h | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lld/wasm/InputChunks.h b/lld/wasm/InputChunks.h index 1fe78d76631f1..762edb61544e2 100644 --- a/lld/wasm/InputChunks.h +++ b/lld/wasm/InputChunks.h @@ -249,7 +249,7 @@ class SyntheticMergedChunk : public InputChunk { std::vector chunks; }; -// Represents a single wasm function within and input file. These are +// Represents a single wasm function within an input file. These are // combined to create the final output CODE section. class InputFunction : public InputChunk { public: diff --git a/lld/wasm/SyntheticSections.h b/lld/wasm/SyntheticSections.h index 2c391d2815914..419bc8e86edc7 100644 --- a/lld/wasm/SyntheticSections.h +++ b/lld/wasm/SyntheticSections.h @@ -501,7 +501,7 @@ class BranchHintSection : public SyntheticSection { std::vector funcHintsVec; - ArrayRef segments; + ArrayRef segments; // XXX }; // Linker generated output sections diff --git a/llvm/include/llvm/ObjectYAML/WasmYAML.h b/llvm/include/llvm/ObjectYAML/WasmYAML.h index b0ab04d1d8ac3..3821b3dedc366 100644 --- a/llvm/include/llvm/ObjectYAML/WasmYAML.h +++ b/llvm/include/llvm/ObjectYAML/WasmYAML.h @@ -247,6 +247,29 @@ struct NameSection : CustomSection { std::vector DataSegmentNames; }; +// A hint for an instruction. +struct HintEntry { + // TODO expr loc + bool likely; +}; + +// All the hints in a function. +struct FuncHintEntry { + StringRef funcName; + std::vector hints; +}; + +struct BranchHintSection : CustomSection { + BranchHintSection() : CustomSection("metadata.code.branch_hint") {} + + static bool classof(const Section *S) { + auto C = dyn_cast(S); + return C && C->Name == "metadata.code.branch_hint"; + } + + std::vector FunctionHints; +}; + struct LinkingSection : CustomSection { LinkingSection() : CustomSection("linking") {} From 70058332b0d5a27b519eb1611b422dc0231ed291 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 08:11:27 -0700 Subject: [PATCH 04/61] Revert "bad" This reverts commit 80210db070d1fd8550f568401d02b46cac45fc95. --- lld/wasm/InputChunks.h | 2 +- lld/wasm/SyntheticSections.h | 2 +- llvm/include/llvm/ObjectYAML/WasmYAML.h | 23 ----------------------- 3 files changed, 2 insertions(+), 25 deletions(-) diff --git a/lld/wasm/InputChunks.h b/lld/wasm/InputChunks.h index 762edb61544e2..1fe78d76631f1 100644 --- a/lld/wasm/InputChunks.h +++ b/lld/wasm/InputChunks.h @@ -249,7 +249,7 @@ class SyntheticMergedChunk : public InputChunk { std::vector chunks; }; -// Represents a single wasm function within an input file. These are +// Represents a single wasm function within and input file. These are // combined to create the final output CODE section. class InputFunction : public InputChunk { public: diff --git a/lld/wasm/SyntheticSections.h b/lld/wasm/SyntheticSections.h index 419bc8e86edc7..2c391d2815914 100644 --- a/lld/wasm/SyntheticSections.h +++ b/lld/wasm/SyntheticSections.h @@ -501,7 +501,7 @@ class BranchHintSection : public SyntheticSection { std::vector funcHintsVec; - ArrayRef segments; // XXX + ArrayRef segments; }; // Linker generated output sections diff --git a/llvm/include/llvm/ObjectYAML/WasmYAML.h b/llvm/include/llvm/ObjectYAML/WasmYAML.h index 3821b3dedc366..b0ab04d1d8ac3 100644 --- a/llvm/include/llvm/ObjectYAML/WasmYAML.h +++ b/llvm/include/llvm/ObjectYAML/WasmYAML.h @@ -247,29 +247,6 @@ struct NameSection : CustomSection { std::vector DataSegmentNames; }; -// A hint for an instruction. -struct HintEntry { - // TODO expr loc - bool likely; -}; - -// All the hints in a function. -struct FuncHintEntry { - StringRef funcName; - std::vector hints; -}; - -struct BranchHintSection : CustomSection { - BranchHintSection() : CustomSection("metadata.code.branch_hint") {} - - static bool classof(const Section *S) { - auto C = dyn_cast(S); - return C && C->Name == "metadata.code.branch_hint"; - } - - std::vector FunctionHints; -}; - struct LinkingSection : CustomSection { LinkingSection() : CustomSection("linking") {} From d989dafe86a0403525dfc18973de57c16ca17857 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 08:11:32 -0700 Subject: [PATCH 05/61] Revert "more" This reverts commit 7294127d65a438d391f7736ed40ede8f9d78dcf1. --- lld/wasm/SyntheticSections.cpp | 5 ----- lld/wasm/Writer.cpp | 2 -- 2 files changed, 7 deletions(-) diff --git a/lld/wasm/SyntheticSections.cpp b/lld/wasm/SyntheticSections.cpp index 5a8151d089ad1..76719596c62e0 100644 --- a/lld/wasm/SyntheticSections.cpp +++ b/lld/wasm/SyntheticSections.cpp @@ -960,9 +960,4 @@ void BuildIdSection::writeBuildId(llvm::ArrayRef buf) { memcpy(hashPlaceholderPtr, buf.data(), hashSize); } -void BranchHintSection::writeBody() { - for (const InputFunction *f : out.functionSec->inputFunctions) { - } -} - } // namespace wasm::lld diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp index 24b6cbcca679e..b704677d36c93 100644 --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -558,7 +558,6 @@ void Writer::addSections() { addSection(out.producersSec); addSection(out.targetFeaturesSec); addSection(out.buildIdSec); - addSection(out.branchHintSec); } void Writer::finalizeSections() { @@ -1703,7 +1702,6 @@ void Writer::createSyntheticSectionsPostLayout() { out.dataCountSec = make(segments); out.linkingSec = make(initFunctions, segments); out.nameSec = make(segments); - out.branchHintSec = make(segments); } void Writer::run() { From bf22519613610e291afe96b36535d93be852d0d2 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 08:11:35 -0700 Subject: [PATCH 06/61] Revert "naive start" This reverts commit 552d29cb757c2f24dc31a2edb5caada8249cf5b0. --- lld/wasm/SyntheticSections.h | 38 ------------------------------------ 1 file changed, 38 deletions(-) diff --git a/lld/wasm/SyntheticSections.h b/lld/wasm/SyntheticSections.h index 2c391d2815914..068fbed11f4a7 100644 --- a/lld/wasm/SyntheticSections.h +++ b/lld/wasm/SyntheticSections.h @@ -467,43 +467,6 @@ class BuildIdSection : public SyntheticSection { uint8_t *hashPlaceholderPtr = nullptr; }; -class BranchHintSection : public SyntheticSection { -public: - BranchHintSection(ArrayRef segments) - : SyntheticSection(llvm::wasm::WASM_SEC_CUSTOM, "metadata.code.branch_hint"), - segments(segments) {} - bool isNeeded() const override { - if (ctx.arg.stripAll && !ctx.arg.keepSections.count(name)) - return false; - return numHints() > 0; - } - void writeBody() override; - size_t numHints() const { - size_t total = 0; - for (auto& funcHints : funcHintsVec) { - total += funcHints.hints.size(); - } - return total; - } - -protected: - // A hint for an instruction. - struct Hint { - // TODO expr loc - bool likely; - }; - - // All the hints in a function. - struct FuncHints { - StringRef funcName; - std::vector hints; - }; - - std::vector funcHintsVec; - - ArrayRef segments; -}; - // Linker generated output sections struct OutStruct { DylinkSection *dylinkSec; @@ -523,7 +486,6 @@ struct OutStruct { ProducersSection *producersSec; TargetFeaturesSection *targetFeaturesSec; BuildIdSection *buildIdSec; - BranchHintSection *branchHintSec; }; extern OutStruct out; From 678e2d59da6da09eac6a58eb138894d0be34544d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 09:00:38 -0700 Subject: [PATCH 07/61] newPASS --- llvm/lib/Target/WebAssembly/CMakeLists.txt | 1 + llvm/lib/Target/WebAssembly/WebAssembly.h | 1 + .../WebAssembly/WebAssemblyBranchHint.cpp | 79 +++++++++++++++++++ .../WebAssembly/WebAssemblyTargetMachine.cpp | 3 + 4 files changed, 84 insertions(+) create mode 100644 llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp diff --git a/llvm/lib/Target/WebAssembly/CMakeLists.txt b/llvm/lib/Target/WebAssembly/CMakeLists.txt index 1e83cbeac50d6..c7130b1f0986b 100644 --- a/llvm/lib/Target/WebAssembly/CMakeLists.txt +++ b/llvm/lib/Target/WebAssembly/CMakeLists.txt @@ -18,6 +18,7 @@ add_llvm_target(WebAssemblyCodeGen WebAssemblyAddMissingPrototypes.cpp WebAssemblyArgumentMove.cpp WebAssemblyAsmPrinter.cpp + WebAssemblyBranchHint.cpp WebAssemblyCFGStackify.cpp WebAssemblyCleanCodeAfterTrap.cpp WebAssemblyCFGSort.cpp diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.h b/llvm/lib/Target/WebAssembly/WebAssembly.h index 17481d77c120a..abdbf3e1cc745 100644 --- a/llvm/lib/Target/WebAssembly/WebAssembly.h +++ b/llvm/lib/Target/WebAssembly/WebAssembly.h @@ -31,6 +31,7 @@ ModulePass *createWebAssemblyFixFunctionBitcasts(); FunctionPass *createWebAssemblyOptimizeReturned(); FunctionPass *createWebAssemblyLowerRefTypesIntPtrConv(); FunctionPass *createWebAssemblyRefTypeMem2Local(); +FunctionPass *createWebAssemblyBranchHint(); // ISel and immediate followup passes. FunctionPass *createWebAssemblyISelDag(WebAssemblyTargetMachine &TM, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp new file mode 100644 index 0000000000000..7912aeb4f5029 --- /dev/null +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp @@ -0,0 +1,79 @@ +//===-- WebAssemblyOptimizeReturned.cpp - Optimize "returned" attributes --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Optimize calls with "returned" attributes for WebAssembly. +/// +//===----------------------------------------------------------------------===// + +#include "WebAssembly.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +using namespace llvm; + +#define DEBUG_TYPE "wasm-optimize-returned" + +namespace { +class OptimizeReturned final : public FunctionPass, + public InstVisitor { + StringRef getPassName() const override { + return "WebAssembly Optimize Returned"; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + AU.addRequired(); + AU.addPreserved(); + FunctionPass::getAnalysisUsage(AU); + } + + bool runOnFunction(Function &F) override; + + DominatorTree *DT = nullptr; + +public: + static char ID; + OptimizeReturned() : FunctionPass(ID) {} + + void visitCallBase(CallBase &CB); +}; +} // End anonymous namespace + +char OptimizeReturned::ID = 0; +INITIALIZE_PASS(OptimizeReturned, DEBUG_TYPE, + "Optimize calls with \"returned\" attributes for WebAssembly", + false, false) + +FunctionPass *llvm::createWebAssemblyOptimizeReturned() { + return new OptimizeReturned(); +} + +void OptimizeReturned::visitCallBase(CallBase &CB) { + for (unsigned I = 0, E = CB.arg_size(); I < E; ++I) + if (CB.paramHasAttr(I, Attribute::Returned)) { + Value *Arg = CB.getArgOperand(I); + // Ignore constants, globals, undef, etc. + if (isa(Arg)) + continue; + // Like replaceDominatedUsesWith but using Instruction/Use dominance. + Arg->replaceUsesWithIf(&CB, + [&](Use &U) { return DT->dominates(&CB, U); }); + } +} + +bool OptimizeReturned::runOnFunction(Function &F) { + LLVM_DEBUG(dbgs() << "********** Optimize returned Attributes **********\n" + "********** Function: " + << F.getName() << '\n'); + + DT = &getAnalysis().getDomTree(); + visit(F); + return true; +} diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp index adb446b20ebf5..07ba1db858e58 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -498,6 +498,9 @@ void WebAssemblyPassConfig::addIRPasses() { // Expand indirectbr instructions to switches. addPass(createIndirectBrExpandPass()); + // TODO flag + addPass(createWebAssemblyBranchHint()); + TargetPassConfig::addIRPasses(); } From 7a715cad98a67eda621012329a39ce3fdf9aa60d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 09:06:54 -0700 Subject: [PATCH 08/61] work --- .../WebAssembly/WebAssemblyBranchHint.cpp | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp index 7912aeb4f5029..6cc542b4cb579 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp @@ -1,4 +1,4 @@ -//===-- WebAssemblyOptimizeReturned.cpp - Optimize "returned" attributes --===// +//===-- WebAssemblyBranchHint.cpp - Generate branch hints from LLVM IR --===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -7,24 +7,27 @@ //===----------------------------------------------------------------------===// /// /// \file -/// Optimize calls with "returned" attributes for WebAssembly. +/// Convert LLVM IR branch_weights +/// (https://llvm.org/docs/LangRef.html#branch-weights) into metadata that will +/// then be emitted as a wasm custom section for branch hints +/// (https://github.com/WebAssembly/branch-hinting). /// //===----------------------------------------------------------------------===// #include "WebAssembly.h" -#include "llvm/IR/Dominators.h" +#include "llvm/IR/Dominators.h" // ? #include "llvm/IR/InstVisitor.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; -#define DEBUG_TYPE "wasm-optimize-returned" +#define DEBUG_TYPE "wasm-branch-hint" namespace { -class OptimizeReturned final : public FunctionPass, - public InstVisitor { +class BranchHint final : public FunctionPass, + public InstVisitor { StringRef getPassName() const override { - return "WebAssembly Optimize Returned"; + return "WebAssembly Branch Hint"; } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -40,22 +43,22 @@ class OptimizeReturned final : public FunctionPass, public: static char ID; - OptimizeReturned() : FunctionPass(ID) {} + BranchHint() : FunctionPass(ID) {} void visitCallBase(CallBase &CB); }; } // End anonymous namespace -char OptimizeReturned::ID = 0; -INITIALIZE_PASS(OptimizeReturned, DEBUG_TYPE, - "Optimize calls with \"returned\" attributes for WebAssembly", +char BranchHint::ID = 0; +INITIALIZE_PASS(BranchHint, DEBUG_TYPE, + "Emit WebAssembly branch hints", false, false) -FunctionPass *llvm::createWebAssemblyOptimizeReturned() { - return new OptimizeReturned(); +FunctionPass *llvm::createWebAssemblyBranchHint() { + return new BranchHint(); } -void OptimizeReturned::visitCallBase(CallBase &CB) { +void BranchHint::visitCallBase(CallBase &CB) { for (unsigned I = 0, E = CB.arg_size(); I < E; ++I) if (CB.paramHasAttr(I, Attribute::Returned)) { Value *Arg = CB.getArgOperand(I); @@ -68,8 +71,8 @@ void OptimizeReturned::visitCallBase(CallBase &CB) { } } -bool OptimizeReturned::runOnFunction(Function &F) { - LLVM_DEBUG(dbgs() << "********** Optimize returned Attributes **********\n" +bool BranchHint::runOnFunction(Function &F) { + LLVM_DEBUG(dbgs() << "********** Emit wasm branch hints **********\n" "********** Function: " << F.getName() << '\n'); From 9157b12f183a53c0cba428cd0a06462614fa011e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 09:10:47 -0700 Subject: [PATCH 09/61] work --- llvm/lib/Target/WebAssembly/CMakeLists.txt | 2 +- llvm/lib/Target/WebAssembly/WebAssembly.h | 2 +- ...hHint.cpp => WebAssemblyBranchHinting.cpp} | 20 +++++++++---------- .../WebAssembly/WebAssemblyTargetMachine.cpp | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) rename llvm/lib/Target/WebAssembly/{WebAssemblyBranchHint.cpp => WebAssemblyBranchHinting.cpp} (81%) diff --git a/llvm/lib/Target/WebAssembly/CMakeLists.txt b/llvm/lib/Target/WebAssembly/CMakeLists.txt index c7130b1f0986b..7795d6f9ec974 100644 --- a/llvm/lib/Target/WebAssembly/CMakeLists.txt +++ b/llvm/lib/Target/WebAssembly/CMakeLists.txt @@ -18,7 +18,7 @@ add_llvm_target(WebAssemblyCodeGen WebAssemblyAddMissingPrototypes.cpp WebAssemblyArgumentMove.cpp WebAssemblyAsmPrinter.cpp - WebAssemblyBranchHint.cpp + WebAssemblyBranchHinting.cpp WebAssemblyCFGStackify.cpp WebAssemblyCleanCodeAfterTrap.cpp WebAssemblyCFGSort.cpp diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.h b/llvm/lib/Target/WebAssembly/WebAssembly.h index abdbf3e1cc745..07ff3ec8992ca 100644 --- a/llvm/lib/Target/WebAssembly/WebAssembly.h +++ b/llvm/lib/Target/WebAssembly/WebAssembly.h @@ -31,7 +31,7 @@ ModulePass *createWebAssemblyFixFunctionBitcasts(); FunctionPass *createWebAssemblyOptimizeReturned(); FunctionPass *createWebAssemblyLowerRefTypesIntPtrConv(); FunctionPass *createWebAssemblyRefTypeMem2Local(); -FunctionPass *createWebAssemblyBranchHint(); +FunctionPass *createWebAssemblyBranchHinting(); // ISel and immediate followup passes. FunctionPass *createWebAssemblyISelDag(WebAssemblyTargetMachine &TM, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp similarity index 81% rename from llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp rename to llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index 6cc542b4cb579..bca3658b583ad 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHint.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -1,4 +1,4 @@ -//===-- WebAssemblyBranchHint.cpp - Generate branch hints from LLVM IR --===// +//===-- WebAssemblyBranchHinting.cpp - Emit branch hints from LLVM IR --===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -24,8 +24,8 @@ using namespace llvm; #define DEBUG_TYPE "wasm-branch-hint" namespace { -class BranchHint final : public FunctionPass, - public InstVisitor { +class BranchHinting final : public FunctionPass, + public InstVisitor { StringRef getPassName() const override { return "WebAssembly Branch Hint"; } @@ -43,22 +43,22 @@ class BranchHint final : public FunctionPass, public: static char ID; - BranchHint() : FunctionPass(ID) {} + BranchHinting() : FunctionPass(ID) {} void visitCallBase(CallBase &CB); }; } // End anonymous namespace -char BranchHint::ID = 0; -INITIALIZE_PASS(BranchHint, DEBUG_TYPE, +char BranchHinting::ID = 0; +INITIALIZE_PASS(BranchHinting, DEBUG_TYPE, "Emit WebAssembly branch hints", false, false) -FunctionPass *llvm::createWebAssemblyBranchHint() { - return new BranchHint(); +FunctionPass *llvm::createWebAssemblyBranchHinting() { + return new BranchHinting(); } -void BranchHint::visitCallBase(CallBase &CB) { +void BranchHinting::visitCallBase(CallBase &CB) { for (unsigned I = 0, E = CB.arg_size(); I < E; ++I) if (CB.paramHasAttr(I, Attribute::Returned)) { Value *Arg = CB.getArgOperand(I); @@ -71,7 +71,7 @@ void BranchHint::visitCallBase(CallBase &CB) { } } -bool BranchHint::runOnFunction(Function &F) { +bool BranchHinting::runOnFunction(Function &F) { LLVM_DEBUG(dbgs() << "********** Emit wasm branch hints **********\n" "********** Function: " << F.getName() << '\n'); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp index 07ba1db858e58..2aea0e8e34329 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -499,7 +499,7 @@ void WebAssemblyPassConfig::addIRPasses() { addPass(createIndirectBrExpandPass()); // TODO flag - addPass(createWebAssemblyBranchHint()); + addPass(createWebAssemblyBranchHinting()); TargetPassConfig::addIRPasses(); } From 7a6be7add6b8bb4338ac08f592c35219a0b3a943 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 09:13:25 -0700 Subject: [PATCH 10/61] work --- llvm/lib/Target/WebAssembly/WebAssembly.h | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.h b/llvm/lib/Target/WebAssembly/WebAssembly.h index 07ff3ec8992ca..a331d755b9162 100644 --- a/llvm/lib/Target/WebAssembly/WebAssembly.h +++ b/llvm/lib/Target/WebAssembly/WebAssembly.h @@ -89,6 +89,7 @@ void initializeWebAssemblyRegNumberingPass(PassRegistry &); void initializeWebAssemblyRegStackifyPass(PassRegistry &); void initializeWebAssemblyReplacePhysRegsPass(PassRegistry &); void initializeWebAssemblySetP2AlignOperandsPass(PassRegistry &); +void initializeWebAssemblyBranchHintingPass(PassRegistry &); namespace WebAssembly { enum TargetIndex { From ae889b44d254e0734023536a7e56aa42cde61288 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 09:14:55 -0700 Subject: [PATCH 11/61] work --- .../WebAssembly/WebAssemblyBranchHinting.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index bca3658b583ad..2ead5400b82dc 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -24,8 +24,8 @@ using namespace llvm; #define DEBUG_TYPE "wasm-branch-hint" namespace { -class BranchHinting final : public FunctionPass, - public InstVisitor { +class WebAssemblyBranchHinting final : public FunctionPass, + public InstVisitor { StringRef getPassName() const override { return "WebAssembly Branch Hint"; } @@ -43,22 +43,22 @@ class BranchHinting final : public FunctionPass, public: static char ID; - BranchHinting() : FunctionPass(ID) {} + WebAssemblyBranchHinting() : FunctionPass(ID) {} void visitCallBase(CallBase &CB); }; } // End anonymous namespace -char BranchHinting::ID = 0; -INITIALIZE_PASS(BranchHinting, DEBUG_TYPE, +char WebAssemblyBranchHinting::ID = 0; +INITIALIZE_PASS(WebAssemblyBranchHinting, DEBUG_TYPE, "Emit WebAssembly branch hints", false, false) FunctionPass *llvm::createWebAssemblyBranchHinting() { - return new BranchHinting(); + return new WebAssemblyBranchHinting(); } -void BranchHinting::visitCallBase(CallBase &CB) { +void WebAssemblyBranchHinting::visitCallBase(CallBase &CB) { for (unsigned I = 0, E = CB.arg_size(); I < E; ++I) if (CB.paramHasAttr(I, Attribute::Returned)) { Value *Arg = CB.getArgOperand(I); @@ -71,7 +71,7 @@ void BranchHinting::visitCallBase(CallBase &CB) { } } -bool BranchHinting::runOnFunction(Function &F) { +bool WebAssemblyBranchHinting::runOnFunction(Function &F) { LLVM_DEBUG(dbgs() << "********** Emit wasm branch hints **********\n" "********** Function: " << F.getName() << '\n'); From 88f4554266e4c22f8ad812aa17ae4de8f764a312 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 09:29:15 -0700 Subject: [PATCH 12/61] work --- .../WebAssembly/WebAssemblyBranchHinting.cpp | 34 ++++++------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index 2ead5400b82dc..aca63334c5c61 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -15,10 +15,18 @@ //===----------------------------------------------------------------------===// #include "WebAssembly.h" -#include "llvm/IR/Dominators.h" // ? +#include "llvm/IR/Dominators.h" #include "llvm/IR/InstVisitor.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +//? +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + + using namespace llvm; #define DEBUG_TYPE "wasm-branch-hint" @@ -30,22 +38,13 @@ class WebAssemblyBranchHinting final : public FunctionPass, return "WebAssembly Branch Hint"; } - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesCFG(); - AU.addRequired(); - AU.addPreserved(); - FunctionPass::getAnalysisUsage(AU); - } - bool runOnFunction(Function &F) override; - DominatorTree *DT = nullptr; - public: static char ID; WebAssemblyBranchHinting() : FunctionPass(ID) {} - void visitCallBase(CallBase &CB); + void visitBranchInst(BranchInst &I); }; } // End anonymous namespace @@ -58,17 +57,7 @@ FunctionPass *llvm::createWebAssemblyBranchHinting() { return new WebAssemblyBranchHinting(); } -void WebAssemblyBranchHinting::visitCallBase(CallBase &CB) { - for (unsigned I = 0, E = CB.arg_size(); I < E; ++I) - if (CB.paramHasAttr(I, Attribute::Returned)) { - Value *Arg = CB.getArgOperand(I); - // Ignore constants, globals, undef, etc. - if (isa(Arg)) - continue; - // Like replaceDominatedUsesWith but using Instruction/Use dominance. - Arg->replaceUsesWithIf(&CB, - [&](Use &U) { return DT->dominates(&CB, U); }); - } +void WebAssemblyBranchHinting::visitBranchInst(BranchInst &I) { } bool WebAssemblyBranchHinting::runOnFunction(Function &F) { @@ -76,7 +65,6 @@ bool WebAssemblyBranchHinting::runOnFunction(Function &F) { "********** Function: " << F.getName() << '\n'); - DT = &getAnalysis().getDomTree(); visit(F); return true; } From fd8359dae9fbdeecc6f6cdf2d6857c643bf8da10 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 10:20:00 -0700 Subject: [PATCH 13/61] reading builds? --- .../WebAssembly/WebAssemblyBranchHinting.cpp | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index aca63334c5c61..d2a7f32d9b3a1 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -58,6 +58,46 @@ FunctionPass *llvm::createWebAssemblyBranchHinting() { } void WebAssemblyBranchHinting::visitBranchInst(BranchInst &I) { + // Check for profiling metadata of the right size and contents (beginning with + // "branch_weights"). + MDNode *ProfMD = I.getMetadata(LLVMContext::MD_prof); + if (!ProfMD) + return; + if (ProfMD->getNumOperands() == 0) + return; + MDString *MDName = dyn_cast(ProfMD->getOperand(0)); + if (!MDName || MDName->getString() != "branch_weights") + return; + + // We expect two integers, for the true and false weights. + if (ProfMD->getNumOperands() < 3) + return; + + // Operand indices for weights, assuming no "expected" operand appears before + // them (which we ignore). + unsigned TrueWeightOp = 1; + unsigned FalseWeightOp = 2; + + // Skip "expected", if present. + if (isa(ProfMD->getOperand(1))) { + if (ProfMD->getNumOperands() < 4) + return; + ++TrueWeightOp; + ++FalseWeightOp; + } + + ConstantInt *TrueWeight = + mdconst::extract(ProfMD->getOperand(TrueWeightOp)); + ConstantInt *FalseWeight = + mdconst::extract(ProfMD->getOperand(FalseWeightOp)); + + if (!TrueWeight || !FalseWeight) + return; + + std::cout << "waka " << TrueWeight->getZExtValue() << " : " << FalseWeight->getZExtValue() << '\n'; + + // Generate metadata for wasm. + // TODO } bool WebAssemblyBranchHinting::runOnFunction(Function &F) { From d794e4b7510fc3c0f739598aa988c2333ebe882a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 10:20:41 -0700 Subject: [PATCH 14/61] reading builds --- llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index d2a7f32d9b3a1..01f1c903b2dbe 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -94,7 +94,7 @@ void WebAssemblyBranchHinting::visitBranchInst(BranchInst &I) { if (!TrueWeight || !FalseWeight) return; - std::cout << "waka " << TrueWeight->getZExtValue() << " : " << FalseWeight->getZExtValue() << '\n'; + errs() << "waka " << TrueWeight->getZExtValue() << " : " << FalseWeight->getZExtValue() << '\n'; // Generate metadata for wasm. // TODO From 0b68888d1604c77053bdae7a91af6d3d62de01f0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 10:36:54 -0700 Subject: [PATCH 15/61] debug --- llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index 01f1c903b2dbe..6a3002e480f85 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -94,7 +94,7 @@ void WebAssemblyBranchHinting::visitBranchInst(BranchInst &I) { if (!TrueWeight || !FalseWeight) return; - errs() << "waka " << TrueWeight->getZExtValue() << " : " << FalseWeight->getZExtValue() << '\n'; + errs() << "seeing weights: " << TrueWeight->getZExtValue() << " : " << FalseWeight->getZExtValue() << '\n'; // Generate metadata for wasm. // TODO From 1d122a2eab3ccb4d41b83b373c04e5e1dc51cb7d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 14:46:17 -0700 Subject: [PATCH 16/61] using MachineBasicBlock --- llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp | 2 +- .../WebAssembly/WebAssemblyBranchHinting.cpp | 117 +++++++++--------- .../WebAssembly/WebAssemblyTargetMachine.cpp | 6 +- 3 files changed, 60 insertions(+), 65 deletions(-) diff --git a/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp b/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp index b13394a607f6a..5c1a020299564 100644 --- a/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp +++ b/llvm/lib/Target/Mips/MipsDelaySlotFiller.cpp @@ -875,7 +875,7 @@ MipsDelaySlotFiller::selectSuccBB(MachineBasicBlock &B) const { if (B.succ_empty()) return nullptr; - // Select the successor with the larget edge weight. + // Select the successor with the largest edge weight. auto &Prob = getAnalysis().getMBPI(); MachineBasicBlock *S = *llvm::max_element(B.successors(), [&](const MachineBasicBlock *Dst0, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index 6a3002e480f85..6fd9df7d56ccf 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -14,6 +14,7 @@ /// //===----------------------------------------------------------------------===// + #include "WebAssembly.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/InstVisitor.h" @@ -25,86 +26,80 @@ #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/Pass.h" - - +//? +#include "MCTargetDesc/WebAssemblyMCTargetDesc.h" +#include "WebAssembly.h" +#include "WebAssemblyMachineFunctionInfo.h" +#include "WebAssemblySubtarget.h" +#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; -#define DEBUG_TYPE "wasm-branch-hint" +#define DEBUG_TYPE "wasm-branch-hinting" namespace { -class WebAssemblyBranchHinting final : public FunctionPass, - public InstVisitor { +class WebAssemblyBranchHinting final : public MachineFunctionPass { StringRef getPassName() const override { - return "WebAssembly Branch Hint"; + return "WebAssembly Lower br_unless"; } - bool runOnFunction(Function &F) override; + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } -public: - static char ID; - WebAssemblyBranchHinting() : FunctionPass(ID) {} + bool runOnMachineFunction(MachineFunction &MF) override; - void visitBranchInst(BranchInst &I); +public: + static char ID; // Pass identification, replacement for typeid + WebAssemblyBranchHinting() : MachineFunctionPass(ID) {} }; -} // End anonymous namespace +} // end anonymous namespace char WebAssemblyBranchHinting::ID = 0; INITIALIZE_PASS(WebAssemblyBranchHinting, DEBUG_TYPE, - "Emit WebAssembly branch hints", - false, false) + "Emits branch hints", false, false) FunctionPass *llvm::createWebAssemblyBranchHinting() { return new WebAssemblyBranchHinting(); } -void WebAssemblyBranchHinting::visitBranchInst(BranchInst &I) { - // Check for profiling metadata of the right size and contents (beginning with - // "branch_weights"). - MDNode *ProfMD = I.getMetadata(LLVMContext::MD_prof); - if (!ProfMD) - return; - if (ProfMD->getNumOperands() == 0) - return; - MDString *MDName = dyn_cast(ProfMD->getOperand(0)); - if (!MDName || MDName->getString() != "branch_weights") - return; - - // We expect two integers, for the true and false weights. - if (ProfMD->getNumOperands() < 3) - return; - - // Operand indices for weights, assuming no "expected" operand appears before - // them (which we ignore). - unsigned TrueWeightOp = 1; - unsigned FalseWeightOp = 2; - - // Skip "expected", if present. - if (isa(ProfMD->getOperand(1))) { - if (ProfMD->getNumOperands() < 4) - return; - ++TrueWeightOp; - ++FalseWeightOp; - } - - ConstantInt *TrueWeight = - mdconst::extract(ProfMD->getOperand(TrueWeightOp)); - ConstantInt *FalseWeight = - mdconst::extract(ProfMD->getOperand(FalseWeightOp)); - - if (!TrueWeight || !FalseWeight) - return; - - errs() << "seeing weights: " << TrueWeight->getZExtValue() << " : " << FalseWeight->getZExtValue() << '\n'; - - // Generate metadata for wasm. - // TODO -} - -bool WebAssemblyBranchHinting::runOnFunction(Function &F) { - LLVM_DEBUG(dbgs() << "********** Emit wasm branch hints **********\n" +bool WebAssemblyBranchHinting::runOnMachineFunction(MachineFunction &MF) { + LLVM_DEBUG(dbgs() << "********** Emitting branch hints **********\n" "********** Function: " - << F.getName() << '\n'); + << MF.getName() << '\n'); + + const MachineBranchProbabilityInfo *MBPI = + &getAnalysis().getMBPI(); + + auto &MFI = *MF.getInfo(); + const auto &TII = *MF.getSubtarget().getInstrInfo(); + auto &MRI = MF.getRegInfo(); + + for (auto &MBB : MF) { + for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) { + if (MI.getOpcode() != WebAssembly::BR_UNLESS && + MI.getOpcode() != WebAssembly::BR_IF) + continue; + + // This is a BR. It has two successors, and perhaps branch probability + // info between them. + errs() << MI << '\n'; + assert(MBB.succ_size() == 2); + auto iter = MBB.succ_begin(); + MachineBasicBlock* first = *iter; + iter++; + MachineBasicBlock* second = *iter; + + BranchProbability probFirst = MBPI->getEdgeProbability(&MBB, first); + BranchProbability probSecond = MBPI->getEdgeProbability(&MBB, second); + errs() << probFirst << " : " << probSecond << '\n'; + } + } - visit(F); return true; } diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp index 2aea0e8e34329..03ba2bfa6addb 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -498,9 +498,6 @@ void WebAssemblyPassConfig::addIRPasses() { // Expand indirectbr instructions to switches. addPass(createIndirectBrExpandPass()); - // TODO flag - addPass(createWebAssemblyBranchHinting()); - TargetPassConfig::addIRPasses(); } @@ -642,6 +639,9 @@ void WebAssemblyPassConfig::addPreEmitPass() { if (!WasmDisableExplicitLocals) addPass(createWebAssemblyDebugFixup()); + // TODO flag waka + addPass(createWebAssemblyBranchHinting()); + // Collect information to prepare for MC lowering / asm printing. addPass(createWebAssemblyMCLowerPrePass()); } From 817a8e9218f3da09f9954566067de5b82156b604 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 11 Jun 2025 14:54:53 -0700 Subject: [PATCH 17/61] clean --- llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index 6fd9df7d56ccf..847fa5b67f238 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -43,7 +43,7 @@ using namespace llvm; namespace { class WebAssemblyBranchHinting final : public MachineFunctionPass { StringRef getPassName() const override { - return "WebAssembly Lower br_unless"; + return "WebAssembly emit branch hints"; } void getAnalysisUsage(AnalysisUsage &AU) const override { From 5917102cf52a2ab789ca9009de0b8fec6ba8b427 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 12 Jun 2025 11:04:10 -0700 Subject: [PATCH 18/61] emit a tiny empty branch hint section from AsmPrinter --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 62 +++++++++++++++++++ .../WebAssembly/WebAssemblyBranchHinting.cpp | 4 ++ 2 files changed, 66 insertions(+) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index c61ed3c7d5d81..a6610dbce36c8 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -32,6 +32,7 @@ #include "llvm/BinaryFormat/Wasm.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" @@ -635,6 +636,67 @@ void WebAssemblyAsmPrinter::emitFunctionBodyStart() { getTargetStreamer()->emitLocal(Locals); AsmPrinter::emitFunctionBodyStart(); + + // Consider branch probability info from the MachineFunction. We must scan all + // instructions now so that we can emit the header part of the custom section + // for branch hinting, which contains the function index and also the number + // of hints in the function. + + errs() << "asmPrint\n"; // waka + + const MachineBranchProbabilityInfo *MBPI = + &getAnalysis().getMBPI(); + + size_t NumHints = 0; + + for (auto &MBB : *MF) { + for (MachineInstr &MI : MBB) { + if (MI.getOpcode() != WebAssembly::BR_UNLESS && + MI.getOpcode() != WebAssembly::BR_IF) + continue; + + // This is a BR. It has two successors, and perhaps branch probability + // info between them. + errs() << MI << '\n'; + assert(MBB.succ_size() == 2); + auto iter = MBB.succ_begin(); + MachineBasicBlock* first = *iter; + iter++; + MachineBasicBlock* second = *iter; + + BranchProbability probFirst = MBPI->getEdgeProbability(&MBB, first); + BranchProbability probSecond = MBPI->getEdgeProbability(&MBB, second); + errs() << probFirst << " : " << probSecond << '\n'; + if (probFirst != probSecond) { + ++NumHints; + } + } + } + + if (NumHints) { + // Emit hints for this function. + // XXX One section for all functions, support multiple functions, for now just one + MCSectionWasm *CustomSection = OutContext.getWasmSection( + ".custom_section.metadata.code.branch_hint", + SectionKind::getMetadata()); + OutStreamer->pushSection(); + OutStreamer->switchSection(CustomSection); + + // One function for now FIXME + OutStreamer->emitULEB128IntValue(1); + + // The function index. + OutStreamer->emitValue( + MCSymbolRefExpr::create(getSymbol(&F), WebAssembly::S_FUNCINDEX, OutContext), + 1); // XXX We need an LEB here! But I see no method to emit a symbol as LEB... + + // The number of hints in the function. + OutStreamer->emitULEB128IntValue(0); // FIXME + + OutStreamer->popSection(); + } + + errs() << "end asmPrint\n"; } void WebAssemblyAsmPrinter::emitInstruction(const MachineInstr *MI) { diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index 847fa5b67f238..75acda7cf37b1 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -73,6 +73,8 @@ bool WebAssemblyBranchHinting::runOnMachineFunction(MachineFunction &MF) { "********** Function: " << MF.getName() << '\n'); + errs() << "Pass\n"; + const MachineBranchProbabilityInfo *MBPI = &getAnalysis().getMBPI(); @@ -101,5 +103,7 @@ bool WebAssemblyBranchHinting::runOnMachineFunction(MachineFunction &MF) { } } + errs() << "and Pass\n"; + return true; } From a7b16485e3b3daebaeadd077b96c0d8bb28bc0b5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 12 Jun 2025 16:51:34 -0700 Subject: [PATCH 19/61] start to emit a hint --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 83 +++++++++++++------ .../WebAssembly/WebAssemblyAsmPrinter.h | 1 + 2 files changed, 60 insertions(+), 24 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index a6610dbce36c8..48bc63008659f 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -648,26 +648,34 @@ void WebAssemblyAsmPrinter::emitFunctionBodyStart() { &getAnalysis().getMBPI(); size_t NumHints = 0; - + + // See if there is a branch hint for an instruction, and if so, if it is true + // or false. + auto getBranchHint = [&](MachineInstr& MI) -> std::optional { + if (MI.getOpcode() != WebAssembly::BR_UNLESS && + MI.getOpcode() != WebAssembly::BR_IF) + return {}; + + // This is a BR. It has two successors, and perhaps branch probability + // info between them. + errs() << MI << '\n'; + auto* MBB = MI.getParent(); + assert(MBB->succ_size() == 2); + auto iter = MBB->succ_begin(); + MachineBasicBlock* first = *iter; + iter++; + MachineBasicBlock* second = *iter; + + BranchProbability probFirst = MBPI->getEdgeProbability(MBB, first); + BranchProbability probSecond = MBPI->getEdgeProbability(MBB, second); + if (probFirst == probSecond) + return {}; + return probFirst > probSecond; + }; + for (auto &MBB : *MF) { for (MachineInstr &MI : MBB) { - if (MI.getOpcode() != WebAssembly::BR_UNLESS && - MI.getOpcode() != WebAssembly::BR_IF) - continue; - - // This is a BR. It has two successors, and perhaps branch probability - // info between them. - errs() << MI << '\n'; - assert(MBB.succ_size() == 2); - auto iter = MBB.succ_begin(); - MachineBasicBlock* first = *iter; - iter++; - MachineBasicBlock* second = *iter; - - BranchProbability probFirst = MBPI->getEdgeProbability(&MBB, first); - BranchProbability probSecond = MBPI->getEdgeProbability(&MBB, second); - errs() << probFirst << " : " << probSecond << '\n'; - if (probFirst != probSecond) { + if (getBranchHint(MI)) { ++NumHints; } } @@ -676,11 +684,14 @@ void WebAssemblyAsmPrinter::emitFunctionBodyStart() { if (NumHints) { // Emit hints for this function. // XXX One section for all functions, support multiple functions, for now just one - MCSectionWasm *CustomSection = OutContext.getWasmSection( - ".custom_section.metadata.code.branch_hint", - SectionKind::getMetadata()); + if (!BranchHintSection) { + BranchHintSection = OutContext.getWasmSection( + ".custom_section.metadata.code.branch_hint", + SectionKind::getMetadata()); + } + OutStreamer->pushSection(); - OutStreamer->switchSection(CustomSection); + OutStreamer->switchSection(BranchHintSection); // One function for now FIXME OutStreamer->emitULEB128IntValue(1); @@ -688,10 +699,34 @@ void WebAssemblyAsmPrinter::emitFunctionBodyStart() { // The function index. OutStreamer->emitValue( MCSymbolRefExpr::create(getSymbol(&F), WebAssembly::S_FUNCINDEX, OutContext), - 1); // XXX We need an LEB here! But I see no method to emit a symbol as LEB... + 1); // XXX We need an LEB here! But I see no method to emit a symbol as LEB... do we emit 4 and let the linker fix that up? // The number of hints in the function. - OutStreamer->emitULEB128IntValue(0); // FIXME + OutStreamer->emitULEB128IntValue(NumHints); + + // The hints. + for (auto &MBB : *MF) { + for (MachineInstr &MI : MBB) { + if (auto Hint = getBranchHint(MI)) { + // Create a temp symbol for this instruction, so we can refer to it, + // then emit the instruction's offset in the function using that. + MCSymbol *InstructionSymbol = OutContext.createTempSymbol(); + OutStreamer->emitLabel(InstructionSymbol); + const MCSymbolRefExpr *InstRef = + MCSymbolRefExpr::create(InstructionSymbol, OutContext); + const MCSymbolRefExpr *FuncRef = + MCSymbolRefExpr::create(getSymbol(&F), OutContext); + const MCBinaryExpr *DiffExpr = + MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); + OutStreamer->emitValue(DiffExpr, 4); // Linker will patch up to LEB? + + // Hints are of size 1. + OutStreamer->emitULEB128IntValue(1); + // The hint itself, likely or not. + OutStreamer->emitULEB128IntValue(*Hint); + } + } + } OutStreamer->popSection(); } diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h index 46063bbe0fba1..019fccf60e2c6 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h @@ -27,6 +27,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyAsmPrinter final : public AsmPrinter { const MachineRegisterInfo *MRI; WebAssemblyFunctionInfo *MFI; bool signaturesEmitted = false; + MCSectionWasm *BranchHintSection = nullptr; public: explicit WebAssemblyAsmPrinter(TargetMachine &TM, From 9d25bf3be7e46b6aa34e2ba2d2ddd57e280b0540 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 12 Jun 2025 17:01:27 -0700 Subject: [PATCH 20/61] emit hint from emitInstruction --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 99 +++++++++---------- .../WebAssembly/WebAssemblyAsmPrinter.h | 4 + 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 48bc63008659f..1bce4f29e0368 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -610,6 +610,31 @@ void WebAssemblyAsmPrinter::emitJumpTableInfo() { // Nothing to do; jump tables are incorporated into the instruction stream. } +std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) { + if (MI.getOpcode() != WebAssembly::BR_UNLESS && + MI.getOpcode() != WebAssembly::BR_IF) + return {}; + + // This is a BR. It has two successors, and perhaps branch probability + // info between them. + errs() << MI << '\n'; + auto* MBB = MI.getParent(); + assert(MBB->succ_size() == 2); + auto iter = MBB->succ_begin(); + MachineBasicBlock* first = *iter; + iter++; + MachineBasicBlock* second = *iter; + + const MachineBranchProbabilityInfo *MBPI = + &getAnalysis().getMBPI(); + + BranchProbability probFirst = MBPI->getEdgeProbability(MBB, first); + BranchProbability probSecond = MBPI->getEdgeProbability(MBB, second); + if (probFirst == probSecond) + return {}; + return probFirst > probSecond; +} + void WebAssemblyAsmPrinter::emitFunctionBodyStart() { const Function &F = MF->getFunction(); SmallVector ResultVTs; @@ -644,35 +669,8 @@ void WebAssemblyAsmPrinter::emitFunctionBodyStart() { errs() << "asmPrint\n"; // waka - const MachineBranchProbabilityInfo *MBPI = - &getAnalysis().getMBPI(); - size_t NumHints = 0; - // See if there is a branch hint for an instruction, and if so, if it is true - // or false. - auto getBranchHint = [&](MachineInstr& MI) -> std::optional { - if (MI.getOpcode() != WebAssembly::BR_UNLESS && - MI.getOpcode() != WebAssembly::BR_IF) - return {}; - - // This is a BR. It has two successors, and perhaps branch probability - // info between them. - errs() << MI << '\n'; - auto* MBB = MI.getParent(); - assert(MBB->succ_size() == 2); - auto iter = MBB->succ_begin(); - MachineBasicBlock* first = *iter; - iter++; - MachineBasicBlock* second = *iter; - - BranchProbability probFirst = MBPI->getEdgeProbability(MBB, first); - BranchProbability probSecond = MBPI->getEdgeProbability(MBB, second); - if (probFirst == probSecond) - return {}; - return probFirst > probSecond; - }; - for (auto &MBB : *MF) { for (MachineInstr &MI : MBB) { if (getBranchHint(MI)) { @@ -704,30 +702,6 @@ void WebAssemblyAsmPrinter::emitFunctionBodyStart() { // The number of hints in the function. OutStreamer->emitULEB128IntValue(NumHints); - // The hints. - for (auto &MBB : *MF) { - for (MachineInstr &MI : MBB) { - if (auto Hint = getBranchHint(MI)) { - // Create a temp symbol for this instruction, so we can refer to it, - // then emit the instruction's offset in the function using that. - MCSymbol *InstructionSymbol = OutContext.createTempSymbol(); - OutStreamer->emitLabel(InstructionSymbol); - const MCSymbolRefExpr *InstRef = - MCSymbolRefExpr::create(InstructionSymbol, OutContext); - const MCSymbolRefExpr *FuncRef = - MCSymbolRefExpr::create(getSymbol(&F), OutContext); - const MCBinaryExpr *DiffExpr = - MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); - OutStreamer->emitValue(DiffExpr, 4); // Linker will patch up to LEB? - - // Hints are of size 1. - OutStreamer->emitULEB128IntValue(1); - // The hint itself, likely or not. - OutStreamer->emitULEB128IntValue(*Hint); - } - } - } - OutStreamer->popSection(); } @@ -739,6 +713,29 @@ void WebAssemblyAsmPrinter::emitInstruction(const MachineInstr *MI) { WebAssembly_MC::verifyInstructionPredicates(MI->getOpcode(), Subtarget->getFeatureBits()); + if (auto Hint = getBranchHint(*MI)) { + OutStreamer->pushSection(); + assert(BranchHintSection); + OutStreamer->switchSection(BranchHintSection); + // Create a temp symbol for this instruction, so we can refer to it, + // then emit the instruction's offset in the function using that. + MCSymbol *InstructionSymbol = OutContext.createTempSymbol(); + OutStreamer->emitLabel(InstructionSymbol); + const MCSymbolRefExpr *InstRef = + MCSymbolRefExpr::create(InstructionSymbol, OutContext); + const MCSymbolRefExpr *FuncRef = + MCSymbolRefExpr::create(getSymbol(&MF->getFunction()), OutContext); + const MCBinaryExpr *DiffExpr = + MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); + OutStreamer->emitValue(DiffExpr, 4); // Linker will patch up to LEB? + + // Hints are of size 1. + OutStreamer->emitULEB128IntValue(1); + // The hint itself, likely or not. + OutStreamer->emitULEB128IntValue(*Hint); + OutStreamer->popSection(); + } + switch (MI->getOpcode()) { case WebAssembly::ARGUMENT_i32: case WebAssembly::ARGUMENT_i32_S: diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h index 019fccf60e2c6..c4165fb2861ae 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h @@ -79,6 +79,10 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyAsmPrinter final : public AsmPrinter { bool &InvokeDetected); MCSymbol *getOrCreateWasmSymbol(StringRef Name); void emitDecls(const Module &M); + + // See if there is a branch hint for an instruction, and if so, if it is true + // or false. + std::optional getBranchHint(const MachineInstr& MI); }; } // end namespace llvm From 9e8a8f78eaef4a0eb9d1214f0127a29a7d4d7db7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Jun 2025 08:39:34 -0700 Subject: [PATCH 21/61] gather for the end --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 119 ++++++++---------- .../WebAssembly/WebAssemblyAsmPrinter.h | 17 ++- 2 files changed, 71 insertions(+), 65 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 1bce4f29e0368..9640e23682a68 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -442,6 +442,7 @@ void WebAssemblyAsmPrinter::emitEndOfAsmFile(Module &M) { EmitProducerInfo(M); EmitTargetFeatures(M); EmitFunctionAttributes(M); + EmitBranchHints(M); } void WebAssemblyAsmPrinter::EmitProducerInfo(Module &M) { @@ -600,6 +601,49 @@ void WebAssemblyAsmPrinter::EmitFunctionAttributes(Module &M) { } } +void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { + if (AllFuncBranchHints.empty()) + return; + + MCSectionWasm *BranchHintSection = OutContext.getWasmSection( + ".custom_section.metadata.code.branch_hint", + SectionKind::getMetadata()); + + OutStreamer->pushSection(); + OutStreamer->switchSection(BranchHintSection); + + // Number of functions with hints. + OutStreamer->emitULEB128IntValue(AllFuncBranchHints.size()); + + for (auto& FuncHints : AllFuncBranchHints) { + auto* FuncSymbol = getSymbol(&FuncHints.MF->getFunction()); + // The function index. + OutStreamer->emitValue( + MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext), + 1); // XXX We need an LEB here! But I see no method to emit a symbol as LEB... do we emit 4 and let the linker fix that up? + + // The number of hints in the function. + OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); + + for (auto& Hint : FuncHints.Hints) { + const MCSymbolRefExpr *InstRef = + MCSymbolRefExpr::create(Hint.Label, OutContext); + const MCSymbolRefExpr *FuncRef = + MCSymbolRefExpr::create(FuncSymbol, OutContext); + const MCBinaryExpr *DiffExpr = + MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); + OutStreamer->emitValue(DiffExpr, 1); // TODO 4 and linker will patch up to LEB? + + // Hints are of size 1. + OutStreamer->emitULEB128IntValue(1); + // The hint itself, likely or not. + OutStreamer->emitULEB128IntValue(Hint.IsLikely); + } + } + + OutStreamer->popSection(); +} + void WebAssemblyAsmPrinter::emitConstantPool() { emitDecls(*MMI->getModule()); assert(MF->getConstantPool()->getConstants().empty() && @@ -662,80 +706,27 @@ void WebAssemblyAsmPrinter::emitFunctionBodyStart() { AsmPrinter::emitFunctionBodyStart(); - // Consider branch probability info from the MachineFunction. We must scan all - // instructions now so that we can emit the header part of the custom section - // for branch hinting, which contains the function index and also the number - // of hints in the function. - - errs() << "asmPrint\n"; // waka - - size_t NumHints = 0; - - for (auto &MBB : *MF) { - for (MachineInstr &MI : MBB) { - if (getBranchHint(MI)) { - ++NumHints; - } - } - } - - if (NumHints) { - // Emit hints for this function. - // XXX One section for all functions, support multiple functions, for now just one - if (!BranchHintSection) { - BranchHintSection = OutContext.getWasmSection( - ".custom_section.metadata.code.branch_hint", - SectionKind::getMetadata()); - } - - OutStreamer->pushSection(); - OutStreamer->switchSection(BranchHintSection); - - // One function for now FIXME - OutStreamer->emitULEB128IntValue(1); - - // The function index. - OutStreamer->emitValue( - MCSymbolRefExpr::create(getSymbol(&F), WebAssembly::S_FUNCINDEX, OutContext), - 1); // XXX We need an LEB here! But I see no method to emit a symbol as LEB... do we emit 4 and let the linker fix that up? - - // The number of hints in the function. - OutStreamer->emitULEB128IntValue(NumHints); - - OutStreamer->popSection(); - } - - errs() << "end asmPrint\n"; } void WebAssemblyAsmPrinter::emitInstruction(const MachineInstr *MI) { LLVM_DEBUG(dbgs() << "EmitInstruction: " << *MI << '\n'); - WebAssembly_MC::verifyInstructionPredicates(MI->getOpcode(), - Subtarget->getFeatureBits()); if (auto Hint = getBranchHint(*MI)) { - OutStreamer->pushSection(); - assert(BranchHintSection); - OutStreamer->switchSection(BranchHintSection); - // Create a temp symbol for this instruction, so we can refer to it, - // then emit the instruction's offset in the function using that. + errs() << "emit branch hint for inst!\n"; + // Create a temp symbol for this instruction, so we can refer to it. MCSymbol *InstructionSymbol = OutContext.createTempSymbol(); OutStreamer->emitLabel(InstructionSymbol); - const MCSymbolRefExpr *InstRef = - MCSymbolRefExpr::create(InstructionSymbol, OutContext); - const MCSymbolRefExpr *FuncRef = - MCSymbolRefExpr::create(getSymbol(&MF->getFunction()), OutContext); - const MCBinaryExpr *DiffExpr = - MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); - OutStreamer->emitValue(DiffExpr, 4); // Linker will patch up to LEB? - - // Hints are of size 1. - OutStreamer->emitULEB128IntValue(1); - // The hint itself, likely or not. - OutStreamer->emitULEB128IntValue(*Hint); - OutStreamer->popSection(); + + // Stash the hint for later, on the proper function. + if (AllFuncBranchHints.empty() || AllFuncBranchHints.back().MF != MF) { + AllFuncBranchHints.emplace_back(FuncBranchHints{MF, {}}); + } + AllFuncBranchHints.back().Hints.emplace_back(BranchHint{InstructionSymbol, *Hint}); } + WebAssembly_MC::verifyInstructionPredicates(MI->getOpcode(), + Subtarget->getFeatureBits()); + switch (MI->getOpcode()) { case WebAssembly::ARGUMENT_i32: case WebAssembly::ARGUMENT_i32_S: diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h index c4165fb2861ae..52018210af4ed 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h @@ -27,7 +27,21 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyAsmPrinter final : public AsmPrinter { const MachineRegisterInfo *MRI; WebAssemblyFunctionInfo *MFI; bool signaturesEmitted = false; - MCSectionWasm *BranchHintSection = nullptr; + + // A branch hint for an instruction. We gather them all (& by function) so we + // can emit the section at the end, knowing how many hints are present (which + // must be emitted before the hints, so we can't do it in a streaming manner). + struct BranchHint { + MCSymbol *Label; + bool IsLikely; + }; + + struct FuncBranchHints { + MachineFunction *MF; + std::vector Hints; + }; + + std::vector AllFuncBranchHints; public: explicit WebAssemblyAsmPrinter(TargetMachine &TM, @@ -60,6 +74,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyAsmPrinter final : public AsmPrinter { void EmitProducerInfo(Module &M); void EmitTargetFeatures(Module &M); void EmitFunctionAttributes(Module &M); + void EmitBranchHints(Module &M); void emitSymbolType(const MCSymbolWasm *Sym); void emitGlobalVariable(const GlobalVariable *GV) override; void emitJumpTableInfo() override; From b1a29d2addb07a64feb6d824179c7383505cc0dd Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Jun 2025 08:44:22 -0700 Subject: [PATCH 22/61] fix --- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 7 ++++--- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 9640e23682a68..09e49073f1ab1 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -616,7 +616,7 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { OutStreamer->emitULEB128IntValue(AllFuncBranchHints.size()); for (auto& FuncHints : AllFuncBranchHints) { - auto* FuncSymbol = getSymbol(&FuncHints.MF->getFunction()); + auto* FuncSymbol = getSymbol(FuncHints.F); // The function index. OutStreamer->emitValue( MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext), @@ -718,8 +718,9 @@ void WebAssemblyAsmPrinter::emitInstruction(const MachineInstr *MI) { OutStreamer->emitLabel(InstructionSymbol); // Stash the hint for later, on the proper function. - if (AllFuncBranchHints.empty() || AllFuncBranchHints.back().MF != MF) { - AllFuncBranchHints.emplace_back(FuncBranchHints{MF, {}}); + Function *F = &MF->getFunction(); + if (AllFuncBranchHints.empty() || AllFuncBranchHints.back().F != F) { + AllFuncBranchHints.emplace_back(FuncBranchHints{F, {}}); } AllFuncBranchHints.back().Hints.emplace_back(BranchHint{InstructionSymbol, *Hint}); } diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h index 52018210af4ed..267ffd5ef2579 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h @@ -37,7 +37,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyAsmPrinter final : public AsmPrinter { }; struct FuncBranchHints { - MachineFunction *MF; + Function *F; std::vector Hints; }; From 66ec0df0c0cb89d9c5443e2d4627723c3a633963 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Jun 2025 09:48:51 -0700 Subject: [PATCH 23/61] clean --- llvm/lib/Target/WebAssembly/CMakeLists.txt | 1 - llvm/lib/Target/WebAssembly/WebAssembly.h | 2 - .../WebAssembly/WebAssemblyAsmPrinter.cpp | 4 +- .../WebAssembly/WebAssemblyBranchHinting.cpp | 109 ------------------ .../WebAssembly/WebAssemblyTargetMachine.cpp | 3 - 5 files changed, 2 insertions(+), 117 deletions(-) delete mode 100644 llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp diff --git a/llvm/lib/Target/WebAssembly/CMakeLists.txt b/llvm/lib/Target/WebAssembly/CMakeLists.txt index 7795d6f9ec974..1e83cbeac50d6 100644 --- a/llvm/lib/Target/WebAssembly/CMakeLists.txt +++ b/llvm/lib/Target/WebAssembly/CMakeLists.txt @@ -18,7 +18,6 @@ add_llvm_target(WebAssemblyCodeGen WebAssemblyAddMissingPrototypes.cpp WebAssemblyArgumentMove.cpp WebAssemblyAsmPrinter.cpp - WebAssemblyBranchHinting.cpp WebAssemblyCFGStackify.cpp WebAssemblyCleanCodeAfterTrap.cpp WebAssemblyCFGSort.cpp diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.h b/llvm/lib/Target/WebAssembly/WebAssembly.h index a331d755b9162..17481d77c120a 100644 --- a/llvm/lib/Target/WebAssembly/WebAssembly.h +++ b/llvm/lib/Target/WebAssembly/WebAssembly.h @@ -31,7 +31,6 @@ ModulePass *createWebAssemblyFixFunctionBitcasts(); FunctionPass *createWebAssemblyOptimizeReturned(); FunctionPass *createWebAssemblyLowerRefTypesIntPtrConv(); FunctionPass *createWebAssemblyRefTypeMem2Local(); -FunctionPass *createWebAssemblyBranchHinting(); // ISel and immediate followup passes. FunctionPass *createWebAssemblyISelDag(WebAssemblyTargetMachine &TM, @@ -89,7 +88,6 @@ void initializeWebAssemblyRegNumberingPass(PassRegistry &); void initializeWebAssemblyRegStackifyPass(PassRegistry &); void initializeWebAssemblyReplacePhysRegsPass(PassRegistry &); void initializeWebAssemblySetP2AlignOperandsPass(PassRegistry &); -void initializeWebAssemblyBranchHintingPass(PassRegistry &); namespace WebAssembly { enum TargetIndex { diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 09e49073f1ab1..62048f62f2b62 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -620,7 +620,7 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { // The function index. OutStreamer->emitValue( MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext), - 1); // XXX We need an LEB here! But I see no method to emit a symbol as LEB... do we emit 4 and let the linker fix that up? + 4); // XXX We need an LEB here! But I see no method to emit a symbol as LEB... do we emit 4 and let the linker fix that up? // The number of hints in the function. OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); @@ -632,7 +632,7 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { MCSymbolRefExpr::create(FuncSymbol, OutContext); const MCBinaryExpr *DiffExpr = MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); - OutStreamer->emitValue(DiffExpr, 1); // TODO 4 and linker will patch up to LEB? + OutStreamer->emitValue(DiffExpr, 4); // TODO 4 and linker will patch up to LEB? // Hints are of size 1. OutStreamer->emitULEB128IntValue(1); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp deleted file mode 100644 index 75acda7cf37b1..0000000000000 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ /dev/null @@ -1,109 +0,0 @@ -//===-- WebAssemblyBranchHinting.cpp - Emit branch hints from LLVM IR --===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// Convert LLVM IR branch_weights -/// (https://llvm.org/docs/LangRef.html#branch-weights) into metadata that will -/// then be emitted as a wasm custom section for branch hints -/// (https://github.com/WebAssembly/branch-hinting). -/// -//===----------------------------------------------------------------------===// - - -#include "WebAssembly.h" -#include "llvm/IR/Dominators.h" -#include "llvm/IR/InstVisitor.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -//? -#include "llvm/IR/InstIterator.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Metadata.h" -#include "llvm/IR/Module.h" -#include "llvm/Pass.h" -//? -#include "MCTargetDesc/WebAssemblyMCTargetDesc.h" -#include "WebAssembly.h" -#include "WebAssemblyMachineFunctionInfo.h" -#include "WebAssemblySubtarget.h" -#include "llvm/CodeGen/MachineBranchProbabilityInfo.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -using namespace llvm; - -#define DEBUG_TYPE "wasm-branch-hinting" - -namespace { -class WebAssemblyBranchHinting final : public MachineFunctionPass { - StringRef getPassName() const override { - return "WebAssembly emit branch hints"; - } - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); - AU.setPreservesCFG(); - MachineFunctionPass::getAnalysisUsage(AU); - } - - bool runOnMachineFunction(MachineFunction &MF) override; - -public: - static char ID; // Pass identification, replacement for typeid - WebAssemblyBranchHinting() : MachineFunctionPass(ID) {} -}; -} // end anonymous namespace - -char WebAssemblyBranchHinting::ID = 0; -INITIALIZE_PASS(WebAssemblyBranchHinting, DEBUG_TYPE, - "Emits branch hints", false, false) - -FunctionPass *llvm::createWebAssemblyBranchHinting() { - return new WebAssemblyBranchHinting(); -} - -bool WebAssemblyBranchHinting::runOnMachineFunction(MachineFunction &MF) { - LLVM_DEBUG(dbgs() << "********** Emitting branch hints **********\n" - "********** Function: " - << MF.getName() << '\n'); - - errs() << "Pass\n"; - - const MachineBranchProbabilityInfo *MBPI = - &getAnalysis().getMBPI(); - - auto &MFI = *MF.getInfo(); - const auto &TII = *MF.getSubtarget().getInstrInfo(); - auto &MRI = MF.getRegInfo(); - - for (auto &MBB : MF) { - for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) { - if (MI.getOpcode() != WebAssembly::BR_UNLESS && - MI.getOpcode() != WebAssembly::BR_IF) - continue; - - // This is a BR. It has two successors, and perhaps branch probability - // info between them. - errs() << MI << '\n'; - assert(MBB.succ_size() == 2); - auto iter = MBB.succ_begin(); - MachineBasicBlock* first = *iter; - iter++; - MachineBasicBlock* second = *iter; - - BranchProbability probFirst = MBPI->getEdgeProbability(&MBB, first); - BranchProbability probSecond = MBPI->getEdgeProbability(&MBB, second); - errs() << probFirst << " : " << probSecond << '\n'; - } - } - - errs() << "and Pass\n"; - - return true; -} diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp index 03ba2bfa6addb..adb446b20ebf5 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -639,9 +639,6 @@ void WebAssemblyPassConfig::addPreEmitPass() { if (!WasmDisableExplicitLocals) addPass(createWebAssemblyDebugFixup()); - // TODO flag waka - addPass(createWebAssemblyBranchHinting()); - // Collect information to prepare for MC lowering / asm printing. addPass(createWebAssemblyMCLowerPrePass()); } From 05f533d2c81729fe0d5d48dcf5b2fc11b663d200 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Jun 2025 09:49:29 -0700 Subject: [PATCH 24/61] clean --- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 62048f62f2b62..ac94a2bfa7382 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -661,7 +661,6 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // This is a BR. It has two successors, and perhaps branch probability // info between them. - errs() << MI << '\n'; auto* MBB = MI.getParent(); assert(MBB->succ_size() == 2); auto iter = MBB->succ_begin(); From 698f1b44b17a6590eebae27f9903bd93570fd18d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Jun 2025 10:06:57 -0700 Subject: [PATCH 25/61] leb.all.the.things --- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index ac94a2bfa7382..967c106d17468 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -618,9 +618,8 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { for (auto& FuncHints : AllFuncBranchHints) { auto* FuncSymbol = getSymbol(FuncHints.F); // The function index. - OutStreamer->emitValue( - MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext), - 4); // XXX We need an LEB here! But I see no method to emit a symbol as LEB... do we emit 4 and let the linker fix that up? + OutStreamer->emitULEB128Value( + MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext)); // The number of hints in the function. OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); @@ -632,7 +631,7 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { MCSymbolRefExpr::create(FuncSymbol, OutContext); const MCBinaryExpr *DiffExpr = MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); - OutStreamer->emitValue(DiffExpr, 4); // TODO 4 and linker will patch up to LEB? + OutStreamer->emitULEB128Value(DiffExpr); // Hints are of size 1. OutStreamer->emitULEB128IntValue(1); @@ -671,6 +670,7 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) const MachineBranchProbabilityInfo *MBPI = &getAnalysis().getMBPI(); + // XXX this is wrong, see b.txt BranchProbability probFirst = MBPI->getEdgeProbability(MBB, first); BranchProbability probSecond = MBPI->getEdgeProbability(MBB, second); if (probFirst == probSecond) From dc0b272cf8c2e168e436b2a44ae8866e80ffea7e Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Jun 2025 10:09:36 -0700 Subject: [PATCH 26/61] undo --- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 967c106d17468..84d064bfe3d52 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -617,9 +617,9 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { for (auto& FuncHints : AllFuncBranchHints) { auto* FuncSymbol = getSymbol(FuncHints.F); - // The function index. - OutStreamer->emitULEB128Value( - MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext)); + // The function index. TODO LEB + OutStreamer->emitValue( + MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext), 4); // The number of hints in the function. OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); @@ -631,7 +631,7 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { MCSymbolRefExpr::create(FuncSymbol, OutContext); const MCBinaryExpr *DiffExpr = MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); - OutStreamer->emitULEB128Value(DiffExpr); + OutStreamer->emitValue(DiffExpr, 4); // TODO LEB // Hints are of size 1. OutStreamer->emitULEB128IntValue(1); From 9d58dc97a5a0ad06954bd32cac7fc6b170fbf332 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 13 Jun 2025 13:10:20 -0700 Subject: [PATCH 27/61] wasm-ld: reorder branch hints section earlier --- lld/wasm/Writer.cpp | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp index b704677d36c93..eda6c220876f5 100644 --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -94,6 +94,7 @@ class Writer { void addSections(); void createCustomSections(); + void createBranchHintSection(); void createSyntheticSections(); void createSyntheticSectionsPostLayout(); void finalizeSections(); @@ -164,7 +165,11 @@ void Writer::createCustomSections() { log("createCustomSections"); for (auto &pair : customSectionMapping) { StringRef name = pair.first; - LLVM_DEBUG(dbgs() << "createCustomSection: " << name << "\n"); + + if (name == "metadata.code.branch_hint") + continue; + + dbgs() << "createCustomSection: " << name << "\n"; OutputSection *sec = make(std::string(name), pair.second); if (ctx.arg.relocatable || ctx.arg.emitRelocs) { @@ -176,6 +181,25 @@ void Writer::createCustomSections() { } } +void Writer::createBranchHintSection() { + StringRef name = "metadata.code.branch_hint"; + auto iter = customSectionMapping.find(name); + if (iter == customSectionMapping.end()) + return; + auto& inputChunks = iter->second; + + dbgs() << "createBranchHintSection: " << name << "\n"; + + OutputSection *sec = make(std::string(name), inputChunks); + auto *sym = make(sec); + out.linkingSec->addToSymtab(sym); + sec->sectionSym = sym; + addSection(sec); + + // Avoid processing it again in createCustomSections. + customSectionMapping.erase("branch_hint"); +} + // Create relocations sections in the final output. // These are only created when relocatable output is requested. void Writer::createRelocSections() { @@ -543,6 +567,7 @@ void Writer::addSections() { addSection(out.startSec); addSection(out.elemSec); addSection(out.dataCountSec); + createBranchHintSection(); addSection(make(out.functionSec->inputFunctions)); addSection(make(segments)); From 99186de40c201854672ce3cf8924c619ccd9ed81 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 16 Jun 2025 16:46:11 -0700 Subject: [PATCH 28/61] lower-level API, emit fixups --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 63 ++++++++++++++----- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 84d064bfe3d52..0e7daa42a3173 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -13,6 +13,11 @@ /// //===----------------------------------------------------------------------===// +#include "llvm/MC/MCObjectStreamer.h" +#include "llvm/MC/MCFragment.h" +#include "MCTargetDesc/WebAssemblyFixupKinds.h" +#include "llvm/MC/MCWasmStreamer.h" + #include "WebAssemblyAsmPrinter.h" #include "MCTargetDesc/WebAssemblyMCExpr.h" #include "MCTargetDesc/WebAssemblyMCTargetDesc.h" @@ -601,13 +606,19 @@ void WebAssemblyAsmPrinter::EmitFunctionAttributes(Module &M) { } } + void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { if (AllFuncBranchHints.empty()) return; + // TODO: Check if safe! XXX + MCObjectStreamer *ObjStreamer = static_cast(OutStreamer.get()); + if (!ObjStreamer) + return; + MCSectionWasm *BranchHintSection = OutContext.getWasmSection( - ".custom_section.metadata.code.branch_hint", - SectionKind::getMetadata()); + ".custom_section.metadata.code.branch_hint", + SectionKind::getMetadata()); OutStreamer->pushSection(); OutStreamer->switchSection(BranchHintSection); @@ -615,25 +626,47 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { // Number of functions with hints. OutStreamer->emitULEB128IntValue(AllFuncBranchHints.size()); + const MCFixupKind FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32); + const unsigned LEB128PadSize = 5; + for (auto& FuncHints : AllFuncBranchHints) { auto* FuncSymbol = getSymbol(FuncHints.F); - // The function index. TODO LEB - OutStreamer->emitValue( - MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext), 4); - // The number of hints in the function. + // Manually emit the relocatable function index. + { + OutStreamer->emitULEB128IntValue(0x42); // waka debug + MCDataFragment *DF = ObjStreamer->getOrCreateDataFragment(); + const MCSymbolRefExpr *FuncIndexExpr = + MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext); + uint64_t Offset = DF->getContents().size(); + DF->getFixups().push_back( + MCFixup::create(Offset, FuncIndexExpr, FixupKind)); + DF->getContents().append(LEB128PadSize, '\0'); + OutStreamer->emitULEB128IntValue(0x43); // waka debug + } + OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); for (auto& Hint : FuncHints.Hints) { - const MCSymbolRefExpr *InstRef = - MCSymbolRefExpr::create(Hint.Label, OutContext); - const MCSymbolRefExpr *FuncRef = - MCSymbolRefExpr::create(FuncSymbol, OutContext); - const MCBinaryExpr *DiffExpr = - MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); - OutStreamer->emitValue(DiffExpr, 4); // TODO LEB - - // Hints are of size 1. + // Manually emit the relocatable instruction offset. + { + OutStreamer->emitULEB128IntValue(0x44); // waka debug + MCDataFragment *DF = ObjStreamer->getOrCreateDataFragment(); + const MCSymbolRefExpr *InstRef = + MCSymbolRefExpr::create(Hint.Label, OutContext); + const MCSymbolRefExpr *FuncRef = + MCSymbolRefExpr::create(FuncSymbol, OutContext); + const MCBinaryExpr *DiffExpr = + MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); + + uint64_t Offset = DF->getContents().size(); + DF->getFixups().push_back( + MCFixup::create(Offset, DiffExpr, FixupKind)); + DF->getContents().append(LEB128PadSize, '\0'); + OutStreamer->emitULEB128IntValue(0x45); // waka debug + } + + // Hint size, always 1 for now. OutStreamer->emitULEB128IntValue(1); // The hint itself, likely or not. OutStreamer->emitULEB128IntValue(Hint.IsLikely); From 45899d5c9ce49e4ae791926d3929eb3bd06eddfe Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 17 Jun 2025 13:17:45 -0700 Subject: [PATCH 29/61] Revert "lower-level API, emit fixups" This reverts commit 99186de40c201854672ce3cf8924c619ccd9ed81. --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 63 +++++-------------- 1 file changed, 15 insertions(+), 48 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 0e7daa42a3173..84d064bfe3d52 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -13,11 +13,6 @@ /// //===----------------------------------------------------------------------===// -#include "llvm/MC/MCObjectStreamer.h" -#include "llvm/MC/MCFragment.h" -#include "MCTargetDesc/WebAssemblyFixupKinds.h" -#include "llvm/MC/MCWasmStreamer.h" - #include "WebAssemblyAsmPrinter.h" #include "MCTargetDesc/WebAssemblyMCExpr.h" #include "MCTargetDesc/WebAssemblyMCTargetDesc.h" @@ -606,19 +601,13 @@ void WebAssemblyAsmPrinter::EmitFunctionAttributes(Module &M) { } } - void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { if (AllFuncBranchHints.empty()) return; - // TODO: Check if safe! XXX - MCObjectStreamer *ObjStreamer = static_cast(OutStreamer.get()); - if (!ObjStreamer) - return; - MCSectionWasm *BranchHintSection = OutContext.getWasmSection( - ".custom_section.metadata.code.branch_hint", - SectionKind::getMetadata()); + ".custom_section.metadata.code.branch_hint", + SectionKind::getMetadata()); OutStreamer->pushSection(); OutStreamer->switchSection(BranchHintSection); @@ -626,47 +615,25 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { // Number of functions with hints. OutStreamer->emitULEB128IntValue(AllFuncBranchHints.size()); - const MCFixupKind FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32); - const unsigned LEB128PadSize = 5; - for (auto& FuncHints : AllFuncBranchHints) { auto* FuncSymbol = getSymbol(FuncHints.F); + // The function index. TODO LEB + OutStreamer->emitValue( + MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext), 4); - // Manually emit the relocatable function index. - { - OutStreamer->emitULEB128IntValue(0x42); // waka debug - MCDataFragment *DF = ObjStreamer->getOrCreateDataFragment(); - const MCSymbolRefExpr *FuncIndexExpr = - MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext); - uint64_t Offset = DF->getContents().size(); - DF->getFixups().push_back( - MCFixup::create(Offset, FuncIndexExpr, FixupKind)); - DF->getContents().append(LEB128PadSize, '\0'); - OutStreamer->emitULEB128IntValue(0x43); // waka debug - } - + // The number of hints in the function. OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); for (auto& Hint : FuncHints.Hints) { - // Manually emit the relocatable instruction offset. - { - OutStreamer->emitULEB128IntValue(0x44); // waka debug - MCDataFragment *DF = ObjStreamer->getOrCreateDataFragment(); - const MCSymbolRefExpr *InstRef = - MCSymbolRefExpr::create(Hint.Label, OutContext); - const MCSymbolRefExpr *FuncRef = - MCSymbolRefExpr::create(FuncSymbol, OutContext); - const MCBinaryExpr *DiffExpr = - MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); - - uint64_t Offset = DF->getContents().size(); - DF->getFixups().push_back( - MCFixup::create(Offset, DiffExpr, FixupKind)); - DF->getContents().append(LEB128PadSize, '\0'); - OutStreamer->emitULEB128IntValue(0x45); // waka debug - } - - // Hint size, always 1 for now. + const MCSymbolRefExpr *InstRef = + MCSymbolRefExpr::create(Hint.Label, OutContext); + const MCSymbolRefExpr *FuncRef = + MCSymbolRefExpr::create(FuncSymbol, OutContext); + const MCBinaryExpr *DiffExpr = + MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); + OutStreamer->emitValue(DiffExpr, 4); // TODO LEB + + // Hints are of size 1. OutStreamer->emitULEB128IntValue(1); // The hint itself, likely or not. OutStreamer->emitULEB128IntValue(Hint.IsLikely); From 56005e7ce3ebc2e4daaf390036bdac51e09a8863 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 17 Jun 2025 14:28:41 -0700 Subject: [PATCH 30/61] horrible hacks but the final binary seems valid --- llvm/lib/MC/MCObjectStreamer.cpp | 27 ++++++++++++++++++- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 13 ++++++--- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index e3d5a5a9a1327..c65eaf6062055 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -6,6 +6,9 @@ // //===----------------------------------------------------------------------===// +// waka XXX see below +#include "../lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h" + #include "llvm/MC/MCObjectStreamer.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAsmInfo.h" @@ -260,12 +263,34 @@ void MCObjectStreamer::emitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, } void MCObjectStreamer::emitULEB128Value(const MCExpr *Value) { + errs() << "emitULEB128\n"; int64_t IntValue; + // Avoid fixups when possible. if (Value->evaluateAsAbsolute(IntValue, getAssemblerPtr())) { + errs() << " absolute\n"; emitULEB128IntValue(IntValue); return; } - insert(getContext().allocFragment(*Value, false)); + + // Old code: + // insert(getContext().allocFragment(*Value, false)); + + errs() << " relative\n"; + + // XXX waka the below should maybe be something like + // std::optional MaybeKind = + // Assembler->getBackend().getFixupKind("fixup_uleb128_i32"); + // rather than hardcode the value, but that does not work (always returns + // false). But also, we really just need to add parameters I guess, for the + // padded size and the fixup kind. + MCFixupKind FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32); + + unsigned PaddedSize = 5; + + MCDataFragment *DF = getOrCreateDataFragment(); + DF->getFixups().push_back(MCFixup::create( + DF->getContents().size(), Value, FixupKind)); + DF->appendContents(PaddedSize, 0); } void MCObjectStreamer::emitSLEB128Value(const MCExpr *Value) { diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 84d064bfe3d52..858b241734625 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -617,9 +617,14 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { for (auto& FuncHints : AllFuncBranchHints) { auto* FuncSymbol = getSymbol(FuncHints.F); - // The function index. TODO LEB - OutStreamer->emitValue( - MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_FUNCINDEX, OutContext), 4); + // The function index. We use S_None because WasmObjectWriter has + // case WebAssembly::S_FUNCINDEX: + // return wasm::R_WASM_FUNCTION_INDEX_I32; + // i.e. S_FUNCINDEX is always relocated as an I32, but we need an LEB. Using + // None gets us to pick the relocation based on the fixup, and + // MCObjectStreamer will emit a proper LEB fixup for emitULEB128Value. + OutStreamer->emitULEB128Value( + MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_None, OutContext)); // The number of hints in the function. OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); @@ -631,7 +636,7 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { MCSymbolRefExpr::create(FuncSymbol, OutContext); const MCBinaryExpr *DiffExpr = MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); - OutStreamer->emitValue(DiffExpr, 4); // TODO LEB + OutStreamer->emitULEB128Value(DiffExpr); // Hints are of size 1. OutStreamer->emitULEB128IntValue(1); From 7aa75c56c7e7cea4a753df9a2bdf4e18d838c173 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 17 Jun 2025 15:31:20 -0700 Subject: [PATCH 31/61] Remove horrible hacks by changing the core LLVM API in MC*Streamer --- llvm/include/llvm/MC/MCObjectStreamer.h | 4 ++- llvm/include/llvm/MC/MCStreamer.h | 8 ++++- llvm/lib/MC/MCAsmStreamer.cpp | 8 +++-- llvm/lib/MC/MCObjectStreamer.cpp | 29 +++++++------------ llvm/lib/MC/MCStreamer.cpp | 4 ++- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 5 +++- 6 files changed, 34 insertions(+), 24 deletions(-) diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h index c987bc2426e9f..ae473ca49858b 100644 --- a/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/llvm/include/llvm/MC/MCObjectStreamer.h @@ -117,7 +117,9 @@ class MCObjectStreamer : public MCStreamer { const MCExpr *Value) override; void emitValueImpl(const MCExpr *Value, unsigned Size, SMLoc Loc = SMLoc()) override; - void emitULEB128Value(const MCExpr *Value) override; + void emitULEB128Value(const MCExpr *Value, + unsigned PadTo = 0, + std::optional Fixup = {}) override; void emitSLEB128Value(const MCExpr *Value) override; void emitWeakReference(MCSymbol *Alias, const MCSymbol *Target) override; void changeSection(MCSection *Section, uint32_t Subsection = 0) override; diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 8f2e137ea0c84..246beebe4e14e 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -750,7 +750,13 @@ class LLVM_ABI MCStreamer { emitIntValue(Value, Size); } - virtual void emitULEB128Value(const MCExpr *Value); + /// The following parameters can be passed in when the Value is not + /// absolute: + /// \param PadTo - How many (zero) bytes of padding to emit. + /// \param Fixup - The fixup to apply in the linker. + virtual void emitULEB128Value(const MCExpr *Value, + unsigned PadTo = 0, + std::optional Fixup = {}); virtual void emitSLEB128Value(const MCExpr *Value); diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp index da0d99e70d9ea..ca5da734500a6 100644 --- a/llvm/lib/MC/MCAsmStreamer.cpp +++ b/llvm/lib/MC/MCAsmStreamer.cpp @@ -256,7 +256,9 @@ class MCAsmStreamer final : public MCStreamer { void emitIntValueInHex(uint64_t Value, unsigned Size) override; void emitIntValueInHexWithPadding(uint64_t Value, unsigned Size) override; - void emitULEB128Value(const MCExpr *Value) override; + void emitULEB128Value(const MCExpr *Value, + unsigned PadTo, + std::optional Fixup) override; void emitSLEB128Value(const MCExpr *Value) override; @@ -1402,7 +1404,9 @@ void MCAsmStreamer::emitValueImpl(const MCExpr *Value, unsigned Size, } } -void MCAsmStreamer::emitULEB128Value(const MCExpr *Value) { +void MCAsmStreamer::emitULEB128Value(const MCExpr *Value, + unsigned PadTo, + std::optional Fixup) { int64_t IntValue; if (Value->evaluateAsAbsolute(IntValue)) { emitULEB128IntValue(IntValue); diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index c65eaf6062055..3af59b1d926ee 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -6,9 +6,6 @@ // //===----------------------------------------------------------------------===// -// waka XXX see below -#include "../lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h" - #include "llvm/MC/MCObjectStreamer.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAsmInfo.h" @@ -262,7 +259,9 @@ void MCObjectStreamer::emitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, Symbol->setOffset(Offset); } -void MCObjectStreamer::emitULEB128Value(const MCExpr *Value) { +void MCObjectStreamer::emitULEB128Value(const MCExpr *Value, + unsigned PadTo, + std::optional Fixup) { errs() << "emitULEB128\n"; int64_t IntValue; // Avoid fixups when possible. @@ -272,25 +271,19 @@ void MCObjectStreamer::emitULEB128Value(const MCExpr *Value) { return; } - // Old code: - // insert(getContext().allocFragment(*Value, false)); + if (!PadTo || !Fixup) { + // Emit the Value as best we can without padding or the fixup. + insert(getContext().allocFragment(*Value, false)); + return; + } errs() << " relative\n"; - // XXX waka the below should maybe be something like - // std::optional MaybeKind = - // Assembler->getBackend().getFixupKind("fixup_uleb128_i32"); - // rather than hardcode the value, but that does not work (always returns - // false). But also, we really just need to add parameters I guess, for the - // padded size and the fixup kind. - MCFixupKind FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32); - - unsigned PaddedSize = 5; - + // Use the given padding and fixup. MCDataFragment *DF = getOrCreateDataFragment(); DF->getFixups().push_back(MCFixup::create( - DF->getContents().size(), Value, FixupKind)); - DF->appendContents(PaddedSize, 0); + DF->getContents().size(), Value, *Fixup)); + DF->appendContents(PadTo, 0); } void MCObjectStreamer::emitSLEB128Value(const MCExpr *Value) { diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp index d70639b7bfe20..4c3f85eb60f3b 100644 --- a/llvm/lib/MC/MCStreamer.cpp +++ b/llvm/lib/MC/MCStreamer.cpp @@ -1323,7 +1323,9 @@ void MCStreamer::emitBinaryData(StringRef Data) { emitBytes(Data); } void MCStreamer::emitValueImpl(const MCExpr *Value, unsigned Size, SMLoc Loc) { visitUsedExpr(*Value); } -void MCStreamer::emitULEB128Value(const MCExpr *Value) {} +void MCStreamer::emitULEB128Value(const MCExpr *Value, + unsigned PadTo, + std::optional Fixup) {} void MCStreamer::emitSLEB128Value(const MCExpr *Value) {} void MCStreamer::emitFill(const MCExpr &NumBytes, uint64_t Value, SMLoc Loc) {} void MCStreamer::emitFill(const MCExpr &NumValues, int64_t Size, int64_t Expr, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 858b241734625..29be7a2ceea85 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -14,6 +14,7 @@ //===----------------------------------------------------------------------===// #include "WebAssemblyAsmPrinter.h" +#include "MCTargetDesc/WebAssemblyFixupKinds.h" #include "MCTargetDesc/WebAssemblyMCExpr.h" #include "MCTargetDesc/WebAssemblyMCTargetDesc.h" #include "MCTargetDesc/WebAssemblyTargetStreamer.h" @@ -624,7 +625,9 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { // None gets us to pick the relocation based on the fixup, and // MCObjectStreamer will emit a proper LEB fixup for emitULEB128Value. OutStreamer->emitULEB128Value( - MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_None, OutContext)); + MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_None, OutContext), + 5, + MCFixupKind(WebAssembly::fixup_uleb128_i32)); // The number of hints in the function. OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); From 67dec35318b959830ccd6501f6745d9667075693 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Jun 2025 11:19:14 -0700 Subject: [PATCH 32/61] test --- llvm/test/CodeGen/WebAssembly/branch-hints.ll | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 llvm/test/CodeGen/WebAssembly/branch-hints.ll diff --git a/llvm/test/CodeGen/WebAssembly/branch-hints.ll b/llvm/test/CodeGen/WebAssembly/branch-hints.ll new file mode 100644 index 0000000000000..452845894059d --- /dev/null +++ b/llvm/test/CodeGen/WebAssembly/branch-hints.ll @@ -0,0 +1,32 @@ +; RUN: llc -mtriple=wasm32-unknown-unknown -filetype=asm -o - < %s | FileCheck %s + +define i32 @bw_bh_test(i32 %a, i32 %b) { +; The weights 42 (true branch) and 1337 (false branch) mean the false +; branch is significantly more likely. The test should generate a hint of "0" +; (unlikely). +entry: + %1 = icmp ult i32 %a, %b + br i1 %1, label %fail, label %success, !prof !0 + +; CHECK: .section .custom_section.metadata.code.branch_hint,"",@ +; Number of functions with hints. +; CHECK-NEXT: .int8 1 +; Function with the hint. +; CHECK-NEXT: .uleb128 bw_bh_test +; Number of hints in function. +; CHECK-NEXT: .int8 1 +; Offset of the hint. +; CHECK-NEXT: .uleb128 .Ltmp0-bw_bh_test +; Size of the hint. +; CHECK-NEXT: .int8 1 +; Value of the hint. +; CHECK-NEXT: .int8 0 + +fail: + ret i32 -1 + +success: + ret i32 0 +} + +!0 = !{!"branch_weights", i32 42, i32 1337} From 4943a232003605d552ecd8cda644e653ff91bfe3 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Jun 2025 11:30:10 -0700 Subject: [PATCH 33/61] test --- lld/test/wasm/branch-hints.ll | 31 +++++++++++++++++++ llvm/test/CodeGen/WebAssembly/branch-hints.ll | 6 ++++ 2 files changed, 37 insertions(+) create mode 100644 lld/test/wasm/branch-hints.ll diff --git a/lld/test/wasm/branch-hints.ll b/lld/test/wasm/branch-hints.ll new file mode 100644 index 0000000000000..07e950c556f05 --- /dev/null +++ b/lld/test/wasm/branch-hints.ll @@ -0,0 +1,31 @@ + +; RUN: llc -mtriple=wasm32-unknown-unknown -filetype=obj -o %t.o < %s +; RUN: wasm-ld -o %t.wasm %t.o --no-entry --no-gc-sections +; RUN: obj2yaml %t.wasm | FileCheck %s + +define i32 @bw_bh_test(i32 %a, i32 %b) { +; The weights 42 (true branch) and 1337 (false branch) mean the false +; branch is significantly more likely. The test should generate a hint of "0" +; (unlikely). +entry: + %1 = icmp ult i32 %a, %b + br i1 %1, label %fail, label %success, !prof !0 + +fail: + ret i32 -1 + +success: + ret i32 0 +} + +!0 = !{!"branch_weights", i32 42, i32 1337} + +; CHECK: - Type: CUSTOM +; CHECK-NEXT: Name: metadata.code.branch_hint +; CHECK-NEXT: Payload: '01818080800001080100' +; ^^ one function +; ^^^^^^^^^^ LEB of function index 1 +; ^^ one hint in function +; ^^ offset 8 +; ^^ hint size 1 +; ^^ hint value: 0 diff --git a/llvm/test/CodeGen/WebAssembly/branch-hints.ll b/llvm/test/CodeGen/WebAssembly/branch-hints.ll index 452845894059d..a2b5c3b8b62dd 100644 --- a/llvm/test/CodeGen/WebAssembly/branch-hints.ll +++ b/llvm/test/CodeGen/WebAssembly/branch-hints.ll @@ -9,16 +9,22 @@ entry: br i1 %1, label %fail, label %success, !prof !0 ; CHECK: .section .custom_section.metadata.code.branch_hint,"",@ + ; Number of functions with hints. ; CHECK-NEXT: .int8 1 + ; Function with the hint. ; CHECK-NEXT: .uleb128 bw_bh_test + ; Number of hints in function. ; CHECK-NEXT: .int8 1 + ; Offset of the hint. ; CHECK-NEXT: .uleb128 .Ltmp0-bw_bh_test + ; Size of the hint. ; CHECK-NEXT: .int8 1 + ; Value of the hint. ; CHECK-NEXT: .int8 0 From 08f259c3a7ec217955c5a31c75d89aef4af8cd06 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Jun 2025 13:42:32 -0700 Subject: [PATCH 34/61] Move fixup to MCAsmBackend --- llvm/include/llvm/MC/MCAsmBackend.h | 5 +++++ llvm/include/llvm/MC/MCObjectStreamer.h | 3 +-- llvm/include/llvm/MC/MCStreamer.h | 7 +------ llvm/lib/MC/MCAsmStreamer.cpp | 6 ++---- llvm/lib/MC/MCObjectStreamer.cpp | 11 +++++++---- llvm/lib/MC/MCStreamer.cpp | 3 +-- .../MCTargetDesc/WebAssemblyAsmBackend.cpp | 8 ++++++++ llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 3 +-- 8 files changed, 26 insertions(+), 20 deletions(-) diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h index e49e786a10f58..0c88edcb15b78 100644 --- a/llvm/include/llvm/MC/MCAsmBackend.h +++ b/llvm/include/llvm/MC/MCAsmBackend.h @@ -127,6 +127,11 @@ class LLVM_ABI MCAsmBackend { const MCValue &Target, MutableArrayRef Data, uint64_t Value, bool IsResolved) = 0; + /// Given a ULEB128 of a particular padded size, return the fixup for it. + virtual MCFixupKind getULEB128Fixup(unsigned PadTo) const { + llvm_unreachable("Need to implement hook if target has ULEB128 fixups"); + } + /// @} /// \name Target Relaxation Interfaces diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h index ae473ca49858b..443b7eb4a7af8 100644 --- a/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/llvm/include/llvm/MC/MCObjectStreamer.h @@ -118,8 +118,7 @@ class MCObjectStreamer : public MCStreamer { void emitValueImpl(const MCExpr *Value, unsigned Size, SMLoc Loc = SMLoc()) override; void emitULEB128Value(const MCExpr *Value, - unsigned PadTo = 0, - std::optional Fixup = {}) override; + unsigned PadTo = 0) override; void emitSLEB128Value(const MCExpr *Value) override; void emitWeakReference(MCSymbol *Alias, const MCSymbol *Target) override; void changeSection(MCSection *Section, uint32_t Subsection = 0) override; diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h index 246beebe4e14e..8fa56dad45971 100644 --- a/llvm/include/llvm/MC/MCStreamer.h +++ b/llvm/include/llvm/MC/MCStreamer.h @@ -750,13 +750,8 @@ class LLVM_ABI MCStreamer { emitIntValue(Value, Size); } - /// The following parameters can be passed in when the Value is not - /// absolute: - /// \param PadTo - How many (zero) bytes of padding to emit. - /// \param Fixup - The fixup to apply in the linker. virtual void emitULEB128Value(const MCExpr *Value, - unsigned PadTo = 0, - std::optional Fixup = {}); + unsigned PadTo = 0); virtual void emitSLEB128Value(const MCExpr *Value); diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp index ca5da734500a6..bbc72287bf1bc 100644 --- a/llvm/lib/MC/MCAsmStreamer.cpp +++ b/llvm/lib/MC/MCAsmStreamer.cpp @@ -257,8 +257,7 @@ class MCAsmStreamer final : public MCStreamer { void emitIntValueInHexWithPadding(uint64_t Value, unsigned Size) override; void emitULEB128Value(const MCExpr *Value, - unsigned PadTo, - std::optional Fixup) override; + unsigned PadTo) override; void emitSLEB128Value(const MCExpr *Value) override; @@ -1405,8 +1404,7 @@ void MCAsmStreamer::emitValueImpl(const MCExpr *Value, unsigned Size, } void MCAsmStreamer::emitULEB128Value(const MCExpr *Value, - unsigned PadTo, - std::optional Fixup) { + unsigned PadTo) { int64_t IntValue; if (Value->evaluateAsAbsolute(IntValue)) { emitULEB128IntValue(IntValue); diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index 3af59b1d926ee..71900a640bdcf 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -260,8 +260,7 @@ void MCObjectStreamer::emitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, } void MCObjectStreamer::emitULEB128Value(const MCExpr *Value, - unsigned PadTo, - std::optional Fixup) { + unsigned PadTo) { errs() << "emitULEB128\n"; int64_t IntValue; // Avoid fixups when possible. @@ -271,7 +270,7 @@ void MCObjectStreamer::emitULEB128Value(const MCExpr *Value, return; } - if (!PadTo || !Fixup) { + if (!PadTo) { // Emit the Value as best we can without padding or the fixup. insert(getContext().allocFragment(*Value, false)); return; @@ -279,10 +278,14 @@ void MCObjectStreamer::emitULEB128Value(const MCExpr *Value, errs() << " relative\n"; + // Use the proper fixup from the specific assembler backend. + const MCAsmBackend &MAB = getAssembler().getBackend(); + MCFixupKind Fixup = MAB.getULEB128Fixup(PadTo); + // Use the given padding and fixup. MCDataFragment *DF = getOrCreateDataFragment(); DF->getFixups().push_back(MCFixup::create( - DF->getContents().size(), Value, *Fixup)); + DF->getContents().size(), Value, Fixup)); DF->appendContents(PadTo, 0); } diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp index 4c3f85eb60f3b..cce2faf144242 100644 --- a/llvm/lib/MC/MCStreamer.cpp +++ b/llvm/lib/MC/MCStreamer.cpp @@ -1324,8 +1324,7 @@ void MCStreamer::emitValueImpl(const MCExpr *Value, unsigned Size, SMLoc Loc) { visitUsedExpr(*Value); } void MCStreamer::emitULEB128Value(const MCExpr *Value, - unsigned PadTo, - std::optional Fixup) {} + unsigned PadTo) {} void MCStreamer::emitSLEB128Value(const MCExpr *Value) {} void MCStreamer::emitFill(const MCExpr &NumBytes, uint64_t Value, SMLoc Loc) {} void MCStreamer::emitFill(const MCExpr &NumValues, int64_t Size, int64_t Expr, diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp index 7bc672c069476..8cbe0207e8e91 100644 --- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp +++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp @@ -46,6 +46,8 @@ class WebAssemblyAsmBackend final : public MCAsmBackend { bool writeNopData(raw_ostream &OS, uint64_t Count, const MCSubtargetInfo *STI) const override; + + MCFixupKind getULEB128Fixup(unsigned PadTo) const override; }; MCFixupKindInfo @@ -78,6 +80,12 @@ bool WebAssemblyAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count, return true; } +MCFixupKind WebAssemblyAsmBackend::getULEB128Fixup(unsigned PadTo) const { + // Only 32-bit is supported for now, which is padded to 5 bytes. + assert(PadTo == 5); + return MCFixupKind(WebAssembly::fixup_uleb128_i32); +} + void WebAssemblyAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 29be7a2ceea85..f13aa07c29e80 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -626,8 +626,7 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { // MCObjectStreamer will emit a proper LEB fixup for emitULEB128Value. OutStreamer->emitULEB128Value( MCSymbolRefExpr::create(FuncSymbol, WebAssembly::S_None, OutContext), - 5, - MCFixupKind(WebAssembly::fixup_uleb128_i32)); + 5); // The number of hints in the function. OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); From a5eff3f999d89cde59e33671a0b8c2b6d1e1205f Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Jun 2025 15:51:41 -0700 Subject: [PATCH 35/61] Add failing .s test --- llvm/test/CodeGen/WebAssembly/branch-hints.ll | 2 + llvm/test/MC/WebAssembly/branch-hints.s | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 llvm/test/MC/WebAssembly/branch-hints.s diff --git a/llvm/test/CodeGen/WebAssembly/branch-hints.ll b/llvm/test/CodeGen/WebAssembly/branch-hints.ll index a2b5c3b8b62dd..f54bf140277a6 100644 --- a/llvm/test/CodeGen/WebAssembly/branch-hints.ll +++ b/llvm/test/CodeGen/WebAssembly/branch-hints.ll @@ -36,3 +36,5 @@ success: } !0 = !{!"branch_weights", i32 42, i32 1337} + +; TODO: a test that starts as asm, and checks either disassembly/objdump or YAML output. Something like llvm/test/MC/WebAssembly/debuginfo-relocs.s or the similar tests in there. diff --git a/llvm/test/MC/WebAssembly/branch-hints.s b/llvm/test/MC/WebAssembly/branch-hints.s new file mode 100644 index 0000000000000..7aeb6617d562a --- /dev/null +++ b/llvm/test/MC/WebAssembly/branch-hints.s @@ -0,0 +1,38 @@ +# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t.o %s +# RUN: obj2yaml %t.o | FileCheck %s + + .file "c.ll" + .functype bw_bh_test (i32, i32) -> (i32) + .section .text.bw_bh_test,"",@ + .globl bw_bh_test # -- Begin function bw_bh_test + .type bw_bh_test,@function +bw_bh_test: # @bw_bh_test + .functype bw_bh_test (i32, i32) -> (i32) +# %bb.0: + block + local.get 0 + local.get 1 + i32.ge_u +.Ltmp0: + br_if 0 # 0: down to label0 +# %bb.1: # %fail + i32.const -1 + return +.LBB0_2: # %success + end_block # label0: + i32.const 0 + # fallthrough-return + end_function + # -- End function + .section .text.bw_bh_test,"",@ + .section .custom_section.metadata.code.branch_hint,"",@ + .int8 1 + .uleb128 bw_bh_test + .int8 1 + .uleb128 .Ltmp0-bw_bh_test + .int8 1 + .int8 0 + .section .text.bw_bh_test,"",@ + +## Test handling of ULEB128 fields in the branch hints section. +## TODO checks From 3c2e6945faeeef3500ee9c70277004c61466c1a0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 18 Jun 2025 16:13:01 -0700 Subject: [PATCH 36/61] Fix + test for llvm-mc --- llvm/lib/MC/MCAsmStreamer.cpp | 9 ++++++++- .../WebAssembly/AsmParser/WebAssemblyAsmParser.cpp | 9 +++++++++ llvm/test/CodeGen/WebAssembly/branch-hints.ll | 2 +- llvm/test/MC/WebAssembly/branch-hints.s | 11 +++++++++-- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp index bbc72287bf1bc..a0b388610a4f3 100644 --- a/llvm/lib/MC/MCAsmStreamer.cpp +++ b/llvm/lib/MC/MCAsmStreamer.cpp @@ -1410,7 +1410,14 @@ void MCAsmStreamer::emitULEB128Value(const MCExpr *Value, emitULEB128IntValue(IntValue); return; } - OS << "\t.uleb128 "; + if (!PadTo) + OS << "\t.uleb128 "; + else { + // A padding size has been specified. For now, all that is suppored is a 5- + // byte LEB, which is an int32. + assert(PadTo == 5); + OS << "\t.uleb128_int32 "; + } Value->print(OS, MAI); EmitEOL(); } diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp index 9649381f07b14..624ce3c795b8a 100644 --- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp +++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp @@ -1116,6 +1116,15 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser { return expect(AsmToken::EndOfStatement, "EOL"); } + if (DirectiveID.getString() == ".uleb128_i32") { + const MCExpr *Val; + SMLoc End; + if (Parser.parseExpression(Val, End)) + return error("Cannot parse .uleb128_i32 expression: ", Lexer.getTok()); + Out.emitULEB128Value(Val, 5); + return expect(AsmToken::EndOfStatement, "EOL"); + } + if (DirectiveID.getString() == ".asciz") { if (checkDataSection()) return ParseStatus::Failure; diff --git a/llvm/test/CodeGen/WebAssembly/branch-hints.ll b/llvm/test/CodeGen/WebAssembly/branch-hints.ll index f54bf140277a6..9faf03fbd4de0 100644 --- a/llvm/test/CodeGen/WebAssembly/branch-hints.ll +++ b/llvm/test/CodeGen/WebAssembly/branch-hints.ll @@ -14,7 +14,7 @@ entry: ; CHECK-NEXT: .int8 1 ; Function with the hint. -; CHECK-NEXT: .uleb128 bw_bh_test +; CHECK-NEXT: .uleb128_int32 bw_bh_test ; Number of hints in function. ; CHECK-NEXT: .int8 1 diff --git a/llvm/test/MC/WebAssembly/branch-hints.s b/llvm/test/MC/WebAssembly/branch-hints.s index 7aeb6617d562a..6c26ad07cc810 100644 --- a/llvm/test/MC/WebAssembly/branch-hints.s +++ b/llvm/test/MC/WebAssembly/branch-hints.s @@ -27,7 +27,7 @@ bw_bh_test: # @bw_bh_test .section .text.bw_bh_test,"",@ .section .custom_section.metadata.code.branch_hint,"",@ .int8 1 - .uleb128 bw_bh_test + .uleb128_i32 bw_bh_test .int8 1 .uleb128 .Ltmp0-bw_bh_test .int8 1 @@ -35,4 +35,11 @@ bw_bh_test: # @bw_bh_test .section .text.bw_bh_test,"",@ ## Test handling of ULEB128 fields in the branch hints section. -## TODO checks +# CHECK: Name: metadata.code.branch_hint +# CHECK-NEXT: Payload: '01808080800001080100' +# ^^ one function +# ^^^^^^^^^^ LEB of function index - +# ^^ one hint in function +# ^^ offset 8 +# ^^ hint size 1 +# ^^ hint value: 0 From 51f2adace381607f7c8bba57de53600a712f50b3 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Jun 2025 11:31:57 -0700 Subject: [PATCH 37/61] avoid emitting trivial branch hints --- lld/test/wasm/branch-hints.ll | 2 +- llvm/lib/MC/MCObjectStreamer.cpp | 6 ++---- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 20 +++++++++++++++---- llvm/test/CodeGen/WebAssembly/branch-hints.ll | 2 +- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lld/test/wasm/branch-hints.ll b/lld/test/wasm/branch-hints.ll index 07e950c556f05..8fe09de2aadc7 100644 --- a/lld/test/wasm/branch-hints.ll +++ b/lld/test/wasm/branch-hints.ll @@ -18,7 +18,7 @@ success: ret i32 0 } -!0 = !{!"branch_weights", i32 42, i32 1337} +!0 = !{!"branch_weights", i32 1, i32 2000} ; CHECK: - Type: CUSTOM ; CHECK-NEXT: Name: metadata.code.branch_hint diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index 71900a640bdcf..03e3f1a2724bd 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -261,23 +261,21 @@ void MCObjectStreamer::emitLabelAtPos(MCSymbol *Symbol, SMLoc Loc, void MCObjectStreamer::emitULEB128Value(const MCExpr *Value, unsigned PadTo) { - errs() << "emitULEB128\n"; int64_t IntValue; // Avoid fixups when possible. if (Value->evaluateAsAbsolute(IntValue, getAssemblerPtr())) { - errs() << " absolute\n"; emitULEB128IntValue(IntValue); return; } if (!PadTo) { // Emit the Value as best we can without padding or the fixup. + errs() << "waka waka\n"; + abort(); insert(getContext().allocFragment(*Value, false)); return; } - errs() << " relative\n"; - // Use the proper fixup from the specific assembler backend. const MCAsmBackend &MAB = getAssembler().getBackend(); MCFixupKind Fixup = MAB.getULEB128Fixup(PadTo); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index f13aa07c29e80..0b8d4458cff44 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -680,9 +680,21 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // XXX this is wrong, see b.txt BranchProbability probFirst = MBPI->getEdgeProbability(MBB, first); BranchProbability probSecond = MBPI->getEdgeProbability(MBB, second); - if (probFirst == probSecond) - return {}; - return probFirst > probSecond; + errs() << "swaka " << probFirst << " vs " << probSecond << '\n'; + + // Wasm branch hints are boolean, and each one takes space in the binary, so + // we do not want to emit hints for trivial things like 55%/45%. Err on the + // side of caution for now and focus on really powerful hints (such as those + // given by __builtin_expect). + if (probFirst > probSecond * 100) { + errs() << " emit true since " << probFirst << " > " << (probSecond * 100) << '\n'; + return true; + } + if (probSecond > probFirst * 100) { + errs() << " emit false since " << probSecond << " > " << (probFirst * 100) << '\n'; + return false; + } + return {}; } void WebAssemblyAsmPrinter::emitFunctionBodyStart() { @@ -718,7 +730,7 @@ void WebAssemblyAsmPrinter::emitInstruction(const MachineInstr *MI) { LLVM_DEBUG(dbgs() << "EmitInstruction: " << *MI << '\n'); if (auto Hint = getBranchHint(*MI)) { - errs() << "emit branch hint for inst!\n"; + //errs() << "emit branch hint for inst!\n"; // Create a temp symbol for this instruction, so we can refer to it. MCSymbol *InstructionSymbol = OutContext.createTempSymbol(); OutStreamer->emitLabel(InstructionSymbol); diff --git a/llvm/test/CodeGen/WebAssembly/branch-hints.ll b/llvm/test/CodeGen/WebAssembly/branch-hints.ll index 9faf03fbd4de0..b0dbcc2af0c52 100644 --- a/llvm/test/CodeGen/WebAssembly/branch-hints.ll +++ b/llvm/test/CodeGen/WebAssembly/branch-hints.ll @@ -35,6 +35,6 @@ success: ret i32 0 } -!0 = !{!"branch_weights", i32 42, i32 1337} +!0 = !{!"branch_weights", i32 1, i32 2000} ; TODO: a test that starts as asm, and checks either disassembly/objdump or YAML output. Something like llvm/test/MC/WebAssembly/debuginfo-relocs.s or the similar tests in there. From 7d282ea7b51fd2eb79771f9d7fab8343eb91bd00 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Jun 2025 12:50:18 -0700 Subject: [PATCH 38/61] cleanup and emit even fewer hints --- llvm/lib/MC/MCObjectStreamer.cpp | 4 +--- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 6 +++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index 03e3f1a2724bd..b3bab04fc341b 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -269,9 +269,7 @@ void MCObjectStreamer::emitULEB128Value(const MCExpr *Value, } if (!PadTo) { - // Emit the Value as best we can without padding or the fixup. - errs() << "waka waka\n"; - abort(); + // Emit the Value as best we can without padding or a fixup. insert(getContext().allocFragment(*Value, false)); return; } diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 0b8d4458cff44..e0b3096d3772a 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -685,7 +685,11 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // Wasm branch hints are boolean, and each one takes space in the binary, so // we do not want to emit hints for trivial things like 55%/45%. Err on the // side of caution for now and focus on really powerful hints (such as those - // given by __builtin_expect). + // given by __builtin_expect), and ignore hints of 100%/0% (code leading to an + // unreachable; we emit an unreachable for them already, which is good enough + // for both toolchains and VMs). + if (probFirst.isZero() || probSecond.isZero()) + return {}; if (probFirst > probSecond * 100) { errs() << " emit true since " << probFirst << " > " << (probSecond * 100) << '\n'; return true; From e341863e1254440cf68d5c5aac6900e54b70e3c7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Jun 2025 13:55:28 -0700 Subject: [PATCH 39/61] Add a pass to prune branch_weights we don't want --- llvm/lib/Target/WebAssembly/CMakeLists.txt | 1 + llvm/lib/Target/WebAssembly/WebAssembly.h | 2 + .../WebAssembly/WebAssemblyBranchHinting.cpp | 93 +++++++++++++++++++ .../WebAssembly/WebAssemblyTargetMachine.cpp | 3 + 4 files changed, 99 insertions(+) create mode 100644 llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp diff --git a/llvm/lib/Target/WebAssembly/CMakeLists.txt b/llvm/lib/Target/WebAssembly/CMakeLists.txt index 1e83cbeac50d6..7795d6f9ec974 100644 --- a/llvm/lib/Target/WebAssembly/CMakeLists.txt +++ b/llvm/lib/Target/WebAssembly/CMakeLists.txt @@ -18,6 +18,7 @@ add_llvm_target(WebAssemblyCodeGen WebAssemblyAddMissingPrototypes.cpp WebAssemblyArgumentMove.cpp WebAssemblyAsmPrinter.cpp + WebAssemblyBranchHinting.cpp WebAssemblyCFGStackify.cpp WebAssemblyCleanCodeAfterTrap.cpp WebAssemblyCFGSort.cpp diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.h b/llvm/lib/Target/WebAssembly/WebAssembly.h index 17481d77c120a..a331d755b9162 100644 --- a/llvm/lib/Target/WebAssembly/WebAssembly.h +++ b/llvm/lib/Target/WebAssembly/WebAssembly.h @@ -31,6 +31,7 @@ ModulePass *createWebAssemblyFixFunctionBitcasts(); FunctionPass *createWebAssemblyOptimizeReturned(); FunctionPass *createWebAssemblyLowerRefTypesIntPtrConv(); FunctionPass *createWebAssemblyRefTypeMem2Local(); +FunctionPass *createWebAssemblyBranchHinting(); // ISel and immediate followup passes. FunctionPass *createWebAssemblyISelDag(WebAssemblyTargetMachine &TM, @@ -88,6 +89,7 @@ void initializeWebAssemblyRegNumberingPass(PassRegistry &); void initializeWebAssemblyRegStackifyPass(PassRegistry &); void initializeWebAssemblyReplacePhysRegsPass(PassRegistry &); void initializeWebAssemblySetP2AlignOperandsPass(PassRegistry &); +void initializeWebAssemblyBranchHintingPass(PassRegistry &); namespace WebAssembly { enum TargetIndex { diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp new file mode 100644 index 0000000000000..3211662d4a6d5 --- /dev/null +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -0,0 +1,93 @@ +//===-- WebAssemblyBranchHinting.cpp - Emit branch hints from LLVM IR --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Convert LLVM IR branch_weights +/// (https://llvm.org/docs/LangRef.html#branch-weights) into metadata that will +/// then be emitted as a wasm custom section for branch hints +/// (https://github.com/WebAssembly/branch-hinting). +/// +//===----------------------------------------------------------------------===// + +#include "WebAssembly.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +//? +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" + + +using namespace llvm; + +#define DEBUG_TYPE "wasm-branch-hint" + +namespace { +class WebAssemblyBranchHinting final : public FunctionPass, + public InstVisitor { + StringRef getPassName() const override { + return "WebAssembly Branch Hint"; + } + + bool runOnFunction(Function &F) override; + +public: + static char ID; + WebAssemblyBranchHinting() : FunctionPass(ID) {} + + void visitBranchInst(BranchInst &I); +}; +} // End anonymous namespace + +char WebAssemblyBranchHinting::ID = 0; +INITIALIZE_PASS(WebAssemblyBranchHinting, DEBUG_TYPE, + "Emit WebAssembly branch hints", + false, false) + +FunctionPass *llvm::createWebAssemblyBranchHinting() { + return new WebAssemblyBranchHinting(); +} + +void WebAssemblyBranchHinting::visitBranchInst(BranchInst &I) { + I.eraseMetadataIf([](unsigned MDKind, MDNode *Node) { + // Look for profiling metadata. + if (MDKind != LLVMContext::MD_prof) + return false; + + // Check for profiling metadata of "branch_weights". + if (Node->getNumOperands() == 0) + return false; + MDString *MDName = dyn_cast(Node->getOperand(0)); + if (!MDName || MDName->getString() != "branch_weights") + return false; + + // This is a branch weights metadata. Keep it only if it has the "expected" + // string. TODO explain + if (Node->getNumOperands() >= 2) { + MDString *MDName = dyn_cast(Node->getOperand(1)); + if (MDName && MDName->getString() == "expected") + return false; + } + + // Discard anything else. + return true; + }); +} + +bool WebAssemblyBranchHinting::runOnFunction(Function &F) { + LLVM_DEBUG(dbgs() << "********** Emit wasm branch hints **********\n" + "********** Function: " + << F.getName() << '\n'); + + visit(F); + return true; +} diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp index adb446b20ebf5..0a19643c2351f 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -498,6 +498,9 @@ void WebAssemblyPassConfig::addIRPasses() { // Expand indirectbr instructions to switches. addPass(createIndirectBrExpandPass()); + // TODO flag (here and elsewhere) + addPass(createWebAssemblyBranchHinting()); + TargetPassConfig::addIRPasses(); } From 86296cfaefa9607106a57dca50491a04a245cdee Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Jun 2025 14:00:42 -0700 Subject: [PATCH 40/61] test fixes --- lld/test/wasm/branch-hints.ll | 2 +- .../WebAssembly/WebAssemblyBranchHinting.cpp | 14 +++++++------- llvm/test/CodeGen/WebAssembly/branch-hints.ll | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lld/test/wasm/branch-hints.ll b/lld/test/wasm/branch-hints.ll index 8fe09de2aadc7..5b2de4ff63af9 100644 --- a/lld/test/wasm/branch-hints.ll +++ b/lld/test/wasm/branch-hints.ll @@ -18,7 +18,7 @@ success: ret i32 0 } -!0 = !{!"branch_weights", i32 1, i32 2000} +!0 = !{!"branch_weights", !"expected", i32 1, i32 2000} ; CHECK: - Type: CUSTOM ; CHECK-NEXT: Name: metadata.code.branch_hint diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index 3211662d4a6d5..e6558890734e8 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -1,4 +1,4 @@ -//===-- WebAssemblyBranchHinting.cpp - Emit branch hints from LLVM IR --===// +//===-- WebAssemblyBranchHinting.cpp - Filter branch hints from LLVM IR --===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -7,10 +7,10 @@ //===----------------------------------------------------------------------===// /// /// \file -/// Convert LLVM IR branch_weights -/// (https://llvm.org/docs/LangRef.html#branch-weights) into metadata that will -/// then be emitted as a wasm custom section for branch hints -/// (https://github.com/WebAssembly/branch-hinting). +/// Remove LLVM IR branch_weights that we do not want in wasm. Wasm branch hints +/// are boolean, and always add size to the binary, so we only want to keep +/// certainly-useful hints. Specifically, we keep hints that began as +/// __builtin_expect in the source. /// //===----------------------------------------------------------------------===// @@ -50,7 +50,7 @@ class WebAssemblyBranchHinting final : public FunctionPass, char WebAssemblyBranchHinting::ID = 0; INITIALIZE_PASS(WebAssemblyBranchHinting, DEBUG_TYPE, - "Emit WebAssembly branch hints", + "Filter WebAssembly branch hints", false, false) FunctionPass *llvm::createWebAssemblyBranchHinting() { @@ -84,7 +84,7 @@ void WebAssemblyBranchHinting::visitBranchInst(BranchInst &I) { } bool WebAssemblyBranchHinting::runOnFunction(Function &F) { - LLVM_DEBUG(dbgs() << "********** Emit wasm branch hints **********\n" + LLVM_DEBUG(dbgs() << "********** Filter wasm branch hints **********\n" "********** Function: " << F.getName() << '\n'); diff --git a/llvm/test/CodeGen/WebAssembly/branch-hints.ll b/llvm/test/CodeGen/WebAssembly/branch-hints.ll index b0dbcc2af0c52..acf00fb733553 100644 --- a/llvm/test/CodeGen/WebAssembly/branch-hints.ll +++ b/llvm/test/CodeGen/WebAssembly/branch-hints.ll @@ -35,6 +35,6 @@ success: ret i32 0 } -!0 = !{!"branch_weights", i32 1, i32 2000} +!0 = !{!"branch_weights", !"expected", i32 1, i32 2000} ; TODO: a test that starts as asm, and checks either disassembly/objdump or YAML output. Something like llvm/test/MC/WebAssembly/debuginfo-relocs.s or the similar tests in there. From 4b96033a7dcc60ee329ecf5fc35e8b76f9e41095 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Jun 2025 14:01:13 -0700 Subject: [PATCH 41/61] fix --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index e0b3096d3772a..9f9eb2a9d309c 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -680,25 +680,10 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // XXX this is wrong, see b.txt BranchProbability probFirst = MBPI->getEdgeProbability(MBB, first); BranchProbability probSecond = MBPI->getEdgeProbability(MBB, second); - errs() << "swaka " << probFirst << " vs " << probSecond << '\n'; - - // Wasm branch hints are boolean, and each one takes space in the binary, so - // we do not want to emit hints for trivial things like 55%/45%. Err on the - // side of caution for now and focus on really powerful hints (such as those - // given by __builtin_expect), and ignore hints of 100%/0% (code leading to an - // unreachable; we emit an unreachable for them already, which is good enough - // for both toolchains and VMs). - if (probFirst.isZero() || probSecond.isZero()) +//errs() << "waka " << probFirst << " vs " << probSecond << '\n'; + if (probFirst == probSecond) return {}; - if (probFirst > probSecond * 100) { - errs() << " emit true since " << probFirst << " > " << (probSecond * 100) << '\n'; - return true; - } - if (probSecond > probFirst * 100) { - errs() << " emit false since " << probSecond << " > " << (probFirst * 100) << '\n'; - return false; - } - return {}; + return probFirst > probSecond; } void WebAssemblyAsmPrinter::emitFunctionBodyStart() { From 1fab176e0d64e45114fe358a3d0a7d5f832e1d9b Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Jun 2025 14:28:03 -0700 Subject: [PATCH 42/61] horrible hack --- .../Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 16 +++++++++++++++- .../WebAssembly/WebAssemblyBranchHinting.cpp | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 9f9eb2a9d309c..b2362a499edeb 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -680,7 +680,21 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // XXX this is wrong, see b.txt BranchProbability probFirst = MBPI->getEdgeProbability(MBB, first); BranchProbability probSecond = MBPI->getEdgeProbability(MBB, second); -//errs() << "waka " << probFirst << " vs " << probSecond << '\n'; + errs() << "waka " << probFirst << " vs " << probSecond << '\n'; + // Wasm branch hints are boolean, and each one takes space in the binary, so + // we do not want to emit hints for trivial things like 55%/45%. Err on the + // side of caution for now and focus on really powerful hints (such as those + // given by __builtin_expect), and ignore hints of 100%/0% (code leading to an + // unreachable; we emit an unreachable for them already, which is good enough + // for both toolchains and VMs). + // + // We detect __builtin_expected-generated hints as follows. XXX horrible + auto isFromExpected = [](BranchProbability Prob) { + // Such hints appear as pairs of + // 0x00106035 / 0x80000000 = 0.05% , 0x7fef9fcb / 0x80000000 = 99.95% + return (Prob.getNumerator() == 0x00106035 || Prob.getNumerator() == 0x7fef9fcb) && + Prob.getDenominator() == 0x80000000; + }; if (probFirst == probSecond) return {}; return probFirst > probSecond; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp index e6558890734e8..7652b1ded63f1 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyBranchHinting.cpp @@ -75,10 +75,12 @@ void WebAssemblyBranchHinting::visitBranchInst(BranchInst &I) { if (Node->getNumOperands() >= 2) { MDString *MDName = dyn_cast(Node->getOperand(1)); if (MDName && MDName->getString() == "expected") + errs() << "waka keeping: " << *Node << '\n'; return false; } // Discard anything else. + errs() << "waka dumping: " << *Node << '\n'; return true; }); } From c708aa88e5c3ac1bf01b0d9839de5a08d94c28c0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Jun 2025 14:31:39 -0700 Subject: [PATCH 43/61] horribler hack, but wasm tests finally pass --- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index b2362a499edeb..4cfceaf83c3ba 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -687,7 +687,8 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // given by __builtin_expect), and ignore hints of 100%/0% (code leading to an // unreachable; we emit an unreachable for them already, which is good enough // for both toolchains and VMs). - // + if (probFirst == probSecond) + return {}; // We detect __builtin_expected-generated hints as follows. XXX horrible auto isFromExpected = [](BranchProbability Prob) { // Such hints appear as pairs of @@ -695,7 +696,7 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) return (Prob.getNumerator() == 0x00106035 || Prob.getNumerator() == 0x7fef9fcb) && Prob.getDenominator() == 0x80000000; }; - if (probFirst == probSecond) + if (!isFromExpected(probFirst) || !isFromExpected(probSecond)) return {}; return probFirst > probSecond; } From 4784fb477d5287a3267108677414df01bd0226e1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Jun 2025 14:37:12 -0700 Subject: [PATCH 44/61] simplify using emitAbsoluteSymbolDiffAsULEB128 --- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 4cfceaf83c3ba..0549a1cd07848 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -632,13 +632,7 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { OutStreamer->emitULEB128IntValue(FuncHints.Hints.size()); for (auto& Hint : FuncHints.Hints) { - const MCSymbolRefExpr *InstRef = - MCSymbolRefExpr::create(Hint.Label, OutContext); - const MCSymbolRefExpr *FuncRef = - MCSymbolRefExpr::create(FuncSymbol, OutContext); - const MCBinaryExpr *DiffExpr = - MCBinaryExpr::create(MCBinaryExpr::Sub, InstRef, FuncRef, OutContext); - OutStreamer->emitULEB128Value(DiffExpr); + OutStreamer->emitAbsoluteSymbolDiffAsULEB128(Hint.Label, FuncSymbol); // Hints are of size 1. OutStreamer->emitULEB128IntValue(1); From dc305e3c4c173398a0e51e36e1ab2fb2135e0b0d Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 20 Jun 2025 16:55:13 -0700 Subject: [PATCH 45/61] awful attempts to find the proper mbb --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 62 ++++++++++++++----- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 0549a1cd07848..093888b54a0c5 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -655,34 +655,58 @@ void WebAssemblyAsmPrinter::emitJumpTableInfo() { } std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) { - if (MI.getOpcode() != WebAssembly::BR_UNLESS && - MI.getOpcode() != WebAssembly::BR_IF) + // WebAssemblyLowerBrUnless should have run before us, removing all BR_UNLESS, + // which makes things simpler for us here. + assert(MI.getOpcode() != WebAssembly::BR_UNLESS); + + if (MI.getOpcode() != WebAssembly::BR_IF) return {}; // This is a BR. It has two successors, and perhaps branch probability - // info between them. - auto* MBB = MI.getParent(); - assert(MBB->succ_size() == 2); - auto iter = MBB->succ_begin(); - MachineBasicBlock* first = *iter; + // info between them. Finding the successor blocks is not trivial, since we + // run after CFGstackify, and even WebAssemblyInstrInfo::analyzeBranch returns + // that it cannot analyze branch targets. First, find the two successors of + // the parent block of this BR_IF. + auto *ParentMBB = MI.getParent(); + assert(ParentMBB->succ_size() == 2); + auto iter = ParentMBB->succ_begin(); + MachineBasicBlock* MBB1 = *iter; iter++; - MachineBasicBlock* second = *iter; - + MachineBasicBlock* MBB2 = *iter; +errs() << " first : " << *MBB1 << '\n'; +errs() << " second: " << *MBB2 << '\n'; + + // Iterate through the parent's basic blocks (linear time!) to find the + // block right after us, which is the fallthrough. If we branch, it is not to + // there. + auto ParentMBBI = ParentMBB->getIterator(); + ++ParentMBBI; + // A block with a BR_IF must have something after it. + assert(ParentMBBI != MF->end()); + auto *FalseDest = &*ParentMBBI; + MachineBasicBlock *TrueDest = FalseDest == MBB1 ? MBB2 : MBB1; + errs() << "MI: " << MI << '\n'; + errs() << "False : " << *FalseDest << '\n'; + errs() << "TargetMBB: " << *TrueDest << '\n'; + + +for (int i = 0; i < MI.getNumOperands(); i++) errs() << "operand[" << i << ": " << MI.getOperand(i) << '\n'; +assert(TrueDest); const MachineBranchProbabilityInfo *MBPI = &getAnalysis().getMBPI(); - // XXX this is wrong, see b.txt - BranchProbability probFirst = MBPI->getEdgeProbability(MBB, first); - BranchProbability probSecond = MBPI->getEdgeProbability(MBB, second); - errs() << "waka " << probFirst << " vs " << probSecond << '\n'; + BranchProbability ProbTarget = MBPI->getEdgeProbability(ParentMBB, TrueDest); +errs() << "BR_IF: " << MI << " with prob " << ProbTarget << '\n'; + +errs() << " to target " << *TrueDest << '\n'; + + // Wasm branch hints are boolean, and each one takes space in the binary, so // we do not want to emit hints for trivial things like 55%/45%. Err on the // side of caution for now and focus on really powerful hints (such as those // given by __builtin_expect), and ignore hints of 100%/0% (code leading to an // unreachable; we emit an unreachable for them already, which is good enough // for both toolchains and VMs). - if (probFirst == probSecond) - return {}; // We detect __builtin_expected-generated hints as follows. XXX horrible auto isFromExpected = [](BranchProbability Prob) { // Such hints appear as pairs of @@ -690,9 +714,13 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) return (Prob.getNumerator() == 0x00106035 || Prob.getNumerator() == 0x7fef9fcb) && Prob.getDenominator() == 0x80000000; }; - if (!isFromExpected(probFirst) || !isFromExpected(probSecond)) + if (!isFromExpected(ProbTarget)) return {}; - return probFirst > probSecond; + + const BranchProbability Half = BranchProbability(1, 2); + assert(ProbTarget != Half); +errs() << " emit " << (ProbTarget > Half) << '\n'; + return ProbTarget > Half; } void WebAssemblyAsmPrinter::emitFunctionBodyStart() { From 3956ad6e8ddd43fa05639476d383ec87433dc0d8 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 23 Jun 2025 09:18:15 -0700 Subject: [PATCH 46/61] fix tests --- lld/test/wasm/branch-hints.ll | 8 ++++---- .../Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 3 ++- llvm/test/CodeGen/WebAssembly/branch-hints.ll | 15 +++++++++++---- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/lld/test/wasm/branch-hints.ll b/lld/test/wasm/branch-hints.ll index 5b2de4ff63af9..5e5d757df1311 100644 --- a/lld/test/wasm/branch-hints.ll +++ b/lld/test/wasm/branch-hints.ll @@ -4,9 +4,9 @@ ; RUN: obj2yaml %t.wasm | FileCheck %s define i32 @bw_bh_test(i32 %a, i32 %b) { -; The weights 42 (true branch) and 1337 (false branch) mean the false -; branch is significantly more likely. The test should generate a hint of "0" -; (unlikely). +; The weights below mean we are far more likely to go to %fail and return -1. +; Codegen will emit the -1 first, so we emit a hint of 0, below, for the value +; of the hint (as in llvm/test/Codegen/WebAssembly/branch-hints.ll). entry: %1 = icmp ult i32 %a, %b br i1 %1, label %fail, label %success, !prof !0 @@ -18,7 +18,7 @@ success: ret i32 0 } -!0 = !{!"branch_weights", !"expected", i32 1, i32 2000} +!0 = !{!"branch_weights", !"expected", i32 2000, i32 1} ; CHECK: - Type: CUSTOM ; CHECK-NEXT: Name: metadata.code.branch_hint diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 093888b54a0c5..10435bcc41b0f 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -717,9 +717,10 @@ errs() << " to target " << *TrueDest << '\n'; if (!isFromExpected(ProbTarget)) return {}; + const BranchProbability Half = BranchProbability(1, 2); assert(ProbTarget != Half); -errs() << " emit " << (ProbTarget > Half) << '\n'; +errs() << "waka EMITT! " << ProbTarget << " vs " << Half << '\n'; return ProbTarget > Half; } diff --git a/llvm/test/CodeGen/WebAssembly/branch-hints.ll b/llvm/test/CodeGen/WebAssembly/branch-hints.ll index acf00fb733553..9a4eaacbea318 100644 --- a/llvm/test/CodeGen/WebAssembly/branch-hints.ll +++ b/llvm/test/CodeGen/WebAssembly/branch-hints.ll @@ -1,13 +1,20 @@ ; RUN: llc -mtriple=wasm32-unknown-unknown -filetype=asm -o - < %s | FileCheck %s define i32 @bw_bh_test(i32 %a, i32 %b) { -; The weights 42 (true branch) and 1337 (false branch) mean the false -; branch is significantly more likely. The test should generate a hint of "0" -; (unlikely). entry: %1 = icmp ult i32 %a, %b br i1 %1, label %fail, label %success, !prof !0 +; The weights below mean we are far more likely to go to %fail and return -1. +; Codegen will emit the -1 first (the same as appearing here): + +; CHECK: i32.const -1 +; CHECK: i32.const 0 + +; Given that layout, we must emit a hint of 0, below, for the value of the hint: +; the VM can assume the condition of the br_if is likely *false*, which means we +; likely fall through to return -1. + ; CHECK: .section .custom_section.metadata.code.branch_hint,"",@ ; Number of functions with hints. @@ -35,6 +42,6 @@ success: ret i32 0 } -!0 = !{!"branch_weights", !"expected", i32 1, i32 2000} +!0 = !{!"branch_weights", !"expected", i32 2000, i32 1} ; TODO: a test that starts as asm, and checks either disassembly/objdump or YAML output. Something like llvm/test/MC/WebAssembly/debuginfo-relocs.s or the similar tests in there. From ee546251098b3e7afd1f1919cdd51b126bf73566 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 23 Jun 2025 09:23:11 -0700 Subject: [PATCH 47/61] clenaup --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 10435bcc41b0f..2df49a5bcad52 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -664,8 +664,8 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // This is a BR. It has two successors, and perhaps branch probability // info between them. Finding the successor blocks is not trivial, since we - // run after CFGstackify, and even WebAssemblyInstrInfo::analyzeBranch returns - // that it cannot analyze branch targets. First, find the two successors of + // run after CFGstackify (and even WebAssemblyInstrInfo::analyzeBranch returns + // that it cannot analyze branch targets). First, find the two successors of // the parent block of this BR_IF. auto *ParentMBB = MI.getParent(); assert(ParentMBB->succ_size() == 2); @@ -673,33 +673,24 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) MachineBasicBlock* MBB1 = *iter; iter++; MachineBasicBlock* MBB2 = *iter; -errs() << " first : " << *MBB1 << '\n'; -errs() << " second: " << *MBB2 << '\n'; - // Iterate through the parent's basic blocks (linear time!) to find the - // block right after us, which is the fallthrough. If we branch, it is not to - // there. + // See which of the two successors is right after us: the fallthrough. If we + // branch, it is to the other block. So the block right after us is where we + // go if the br_if condition is false. auto ParentMBBI = ParentMBB->getIterator(); ++ParentMBBI; // A block with a BR_IF must have something after it. assert(ParentMBBI != MF->end()); auto *FalseDest = &*ParentMBBI; - MachineBasicBlock *TrueDest = FalseDest == MBB1 ? MBB2 : MBB1; - errs() << "MI: " << MI << '\n'; - errs() << "False : " << *FalseDest << '\n'; - errs() << "TargetMBB: " << *TrueDest << '\n'; + // It must be one of the successors. + assert(FalseDest == MBB1 || FalseDest == MBB2); + // The true destination (i.e. if the condition is true) is the other one. + MachineBasicBlock *TrueDest = (FalseDest == MBB1 ? MBB2 : MBB1); - -for (int i = 0; i < MI.getNumOperands(); i++) errs() << "operand[" << i << ": " << MI.getOperand(i) << '\n'; -assert(TrueDest); + // Find the probability of branching. const MachineBranchProbabilityInfo *MBPI = &getAnalysis().getMBPI(); - BranchProbability ProbTarget = MBPI->getEdgeProbability(ParentMBB, TrueDest); -errs() << "BR_IF: " << MI << " with prob " << ProbTarget << '\n'; - -errs() << " to target " << *TrueDest << '\n'; - // Wasm branch hints are boolean, and each one takes space in the binary, so // we do not want to emit hints for trivial things like 55%/45%. Err on the @@ -717,10 +708,8 @@ errs() << " to target " << *TrueDest << '\n'; if (!isFromExpected(ProbTarget)) return {}; - const BranchProbability Half = BranchProbability(1, 2); assert(ProbTarget != Half); -errs() << "waka EMITT! " << ProbTarget << " vs " << Half << '\n'; return ProbTarget > Half; } From 8b602613e62816ba860f5f08596eca70e212b704 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 23 Jun 2025 13:04:49 -0700 Subject: [PATCH 48/61] typo --- llvm/lib/MC/MCAsmStreamer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp index a0b388610a4f3..17943c7b4227e 100644 --- a/llvm/lib/MC/MCAsmStreamer.cpp +++ b/llvm/lib/MC/MCAsmStreamer.cpp @@ -1413,7 +1413,7 @@ void MCAsmStreamer::emitULEB128Value(const MCExpr *Value, if (!PadTo) OS << "\t.uleb128 "; else { - // A padding size has been specified. For now, all that is suppored is a 5- + // A padding size has been specified. For now, all that is supported is a 5- // byte LEB, which is an int32. assert(PadTo == 5); OS << "\t.uleb128_int32 "; From bb921ee0dd222af177beaf964944d361df8c7ad3 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 23 Jun 2025 13:09:45 -0700 Subject: [PATCH 49/61] cleanup --- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 4 ++-- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h | 4 ++++ llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 2df49a5bcad52..4902e75c65b1c 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -746,12 +746,12 @@ void WebAssemblyAsmPrinter::emitInstruction(const MachineInstr *MI) { LLVM_DEBUG(dbgs() << "EmitInstruction: " << *MI << '\n'); if (auto Hint = getBranchHint(*MI)) { - //errs() << "emit branch hint for inst!\n"; // Create a temp symbol for this instruction, so we can refer to it. MCSymbol *InstructionSymbol = OutContext.createTempSymbol(); OutStreamer->emitLabel(InstructionSymbol); - // Stash the hint for later, on the proper function. + // Stash the hint for later, on the proper function. We will emit the branch + // hints section at the end, when we know all the information we need. Function *F = &MF->getFunction(); if (AllFuncBranchHints.empty() || AllFuncBranchHints.back().F != F) { AllFuncBranchHints.emplace_back(FuncBranchHints{F, {}}); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h index 267ffd5ef2579..79cea2c87718a 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h @@ -31,8 +31,12 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyAsmPrinter final : public AsmPrinter { // A branch hint for an instruction. We gather them all (& by function) so we // can emit the section at the end, knowing how many hints are present (which // must be emitted before the hints, so we can't do it in a streaming manner). + // And we must gather during emitInstruction(), as we must emit a label for + // each branch we want to annotate (so we can refer to it). struct BranchHint { + // The location the hint refers to. MCSymbol *Label; + // Whether the branch there is likely. bool IsLikely; }; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp index 0a19643c2351f..7bd91147dc458 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -500,7 +500,7 @@ void WebAssemblyPassConfig::addIRPasses() { // TODO flag (here and elsewhere) addPass(createWebAssemblyBranchHinting()); - + TargetPassConfig::addIRPasses(); } From 96b80cecfb57bc1ddafddedf26e0255ca7a9f093 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 23 Jun 2025 13:16:03 -0700 Subject: [PATCH 50/61] comments --- lld/wasm/Writer.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp index eda6c220876f5..928497c9519a3 100644 --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -196,7 +196,9 @@ void Writer::createBranchHintSection() { sec->sectionSym = sym; addSection(sec); - // Avoid processing it again in createCustomSections. + // After emitting this section, avoid processing it again in the place that + // custom sections are normally created, later in the binary (inside + // createCustomSections). customSectionMapping.erase("branch_hint"); } @@ -567,6 +569,9 @@ void Writer::addSections() { addSection(out.startSec); addSection(out.elemSec); addSection(out.dataCountSec); + + // The Branch Hints section is a special custom section that must be emitted + // before the code section. createBranchHintSection(); addSection(make(out.functionSec->inputFunctions)); From 31b8996f3fcc3ee21c5a9d805def049a51d43511 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 23 Jun 2025 16:20:14 -0700 Subject: [PATCH 51/61] wild and misguided effort to merge branch hint sections --- lld/wasm/InputChunks.h | 4 +- lld/wasm/OutputSections.cpp | 62 +++++++++++++++++++ lld/wasm/OutputSections.h | 31 ++++++++++ lld/wasm/Writer.cpp | 6 +- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 26 +++++++- 5 files changed, 123 insertions(+), 6 deletions(-) diff --git a/lld/wasm/InputChunks.h b/lld/wasm/InputChunks.h index 1fe78d76631f1..4c0d1b302b8c2 100644 --- a/lld/wasm/InputChunks.h +++ b/lld/wasm/InputChunks.h @@ -354,9 +354,11 @@ class InputSection : public InputChunk { const uint64_t tombstoneValue; + // XXX + const WasmSection §ion; + protected: static uint64_t getTombstoneForSection(StringRef name); - const WasmSection §ion; }; } // namespace wasm diff --git a/lld/wasm/OutputSections.cpp b/lld/wasm/OutputSections.cpp index 8ccd38f7895cb..2b2f372f84526 100644 --- a/lld/wasm/OutputSections.cpp +++ b/lld/wasm/OutputSections.cpp @@ -282,5 +282,67 @@ void CustomSection::writeRelocations(raw_ostream &os) const { s->writeRelocations(os); } +// Branch Hints + +void BranchHintSection::BranchHintSection(ArrayRef inputSections) : CustomSection("metadata.code.branch_hint", inputSections) { + // Replace the original input sections with artificial ones. The change we + // make is to add the total number of functions across all sections to the + // first section's first 5 bytes, and then to delete the first 5 bytes in all + // subsequent sections. After that, simply concatenating the sections leads to + // the proper output. + assert(!inputSections.empty()); + if (inputSections.size() == 1) { + // Nothing to do. + return; + } + + auto *newInputSections = make>(); + newInputSections.resize(inputSections.size()); + + // Remove the first 5 bytes from all sections but the first, and count how + // many functions there are (so we can add that to the first). + uint64_t totalFunctions = 0; + for (unsigned i = 1; i < inputSections.size(); i++) { + const InputChunk *section = inputSections[i]; + assert(section.kind() == InputChunk::Section); + + // Read the number of functions in this section. + totalFunctions += decodeULEB128(section->wasmSection.Content.data()); + + // Create an adjusted wasm section, without the first 5 bytes. + WasmSection *adjustedWasmSection = make(section->wasmSection); + adjustedWasmSection.Content = adjustedWasmSection.Content.slice(5); + for (auto& relocation : adjustedWasmSection->relocations) + relocation.offset -= 5; + + newInputSections[i] = make(adjustedWasmSection, section->file, section->alignment); + } + + // Add the number of functions to the first section. + { + const InputChunk *section = inputSections[0]; + assert(section.kind() == InputChunk::Section); + + // Read the number of functions in this section. + totalFunctions += decodeULEB128(section->wasmSection.Content.data()); + + // Create an adjusted wasm section, with the first 5 bytes modified so that + // we apply the total number of functions. + WasmSection *adjustedWasmSection = make(section->wasmSection); + adjustedWasmSection.Content = adjustedWasmSection.Content.copy(); + + std::string str; + raw_string_ostream os(str); + encodeULEB128(totalFunctions, os); + // XXX? os << name; + memcpy(adjustedWasmSection.Content.data(), str.data(), 5); + + newInputSections[i] = make(adjustedWasmSection, section->file, section->alignment); + } + + // Use these new artificial sections. + inputSections = newInputSections; +} + } // namespace wasm } // namespace lld diff --git a/lld/wasm/OutputSections.h b/lld/wasm/OutputSections.h index 4b0329dd16cf2..3d46ca6394a6a 100644 --- a/lld/wasm/OutputSections.h +++ b/lld/wasm/OutputSections.h @@ -132,6 +132,37 @@ class CustomSection : public OutputSection { std::string nameData; }; +// A Branch Hint section is a Custom Section with some custom rules for how it +// is created. Rather than simply concatenate the input sections, we must also +// adjust the field that reports the number of functions. +// +// Number of functions with hints. We pad this to 5 bytes to make the linker's +// life easier: given multiple Branch Hint sections, wasm-ld will by default +// simply concatenate them, just like any other custom section. That would end +// up with +// +// [num functions_1] : 5 byte LEB +// [..data_1..] +// [num functions_2] : 5 byte LEB +// [..data_2..] +// .. +// [num functions_N] : 5 byte LEB +// [..data_N..] +// +// To get this to the proper form the linker should emit we can trample the +// number of functions at the very start, and not emit the others: +// +// [num functions_1 + _2 + .. + _N] : 5 byte LEB +// [..data_1..] +// [..data_2..] +// .. +// [..data_N..] +// +class BranchHintSection : public CustomSection { +public: + BranchHintSection(ArrayRef inputSections); +}; + } // namespace wasm } // namespace lld diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp index 928497c9519a3..83138d86528ea 100644 --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -166,7 +166,7 @@ void Writer::createCustomSections() { for (auto &pair : customSectionMapping) { StringRef name = pair.first; - if (name == "metadata.code.branch_hint") + if (name == "metadata.code.branch_hint") // XXX unneeded continue; dbgs() << "createCustomSection: " << name << "\n"; @@ -188,9 +188,9 @@ void Writer::createBranchHintSection() { return; auto& inputChunks = iter->second; - dbgs() << "createBranchHintSection: " << name << "\n"; + dbgs() << "createBranchHintSection!: " << name << "\n"; - OutputSection *sec = make(std::string(name), inputChunks); + OutputSection *sec = make(std::string(name), inputChunks); auto *sym = make(sec); out.linkingSec->addToSymtab(sym); sec->sectionSym = sym; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 4902e75c65b1c..45e1729331d7b 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -613,8 +613,30 @@ void WebAssemblyAsmPrinter::EmitBranchHints(Module &M) { OutStreamer->pushSection(); OutStreamer->switchSection(BranchHintSection); - // Number of functions with hints. - OutStreamer->emitULEB128IntValue(AllFuncBranchHints.size()); + // Number of functions with hints. We pad this to 5 bytes to make the linker's + // life easier: given multiple Branch Hint sections, wasm-ld will by default + // simply concatenate them, just like any other custom section. That would end + // up with + // + // [num functions_1] : 5 byte LEB + // [..data_1..] + // [num functions_2] : 5 byte LEB + // [..data_2..] + // .. + // [num functions_N] : 5 byte LEB + // [..data_N..] + // + // To get this to the proper form the linker should emit we can trample the + // number of functions at the very start, and not emit the others: + // + // [num functions_1 + _2 + .. + _N] : 5 byte LEB + // [..data_1..] + // [..data_2..] + // .. + // [..data_N..] + // + // TODO: fix existing tests for this + OutStreamer->emitULEB128IntValue(AllFuncBranchHints.size(), 5); for (auto& FuncHints : AllFuncBranchHints) { auto* FuncSymbol = getSymbol(FuncHints.F); From 0599b0447ee2ffb6c8fa4876083e43cc65cf2797 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 25 Jun 2025 09:38:25 -0700 Subject: [PATCH 52/61] Handle some corner cases of wasm control flow --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 45e1729331d7b..b65ae94859772 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -684,13 +684,23 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) if (MI.getOpcode() != WebAssembly::BR_IF) return {}; + // We only handle the simple case of our being the last terminator of the + // block. If there are other terminators after us, things may be complicated. + auto *ParentMBB = MI.getParent(); + auto TerminatorRange = MBB.terminators(); + assert(!TerminatorRange.empty()); + auto *LastTerminator = &*TerminatorRange.rbegin(); + if (&MI != LastTerminator) + return {}; + // This is a BR. It has two successors, and perhaps branch probability // info between them. Finding the successor blocks is not trivial, since we // run after CFGstackify (and even WebAssemblyInstrInfo::analyzeBranch returns // that it cannot analyze branch targets). First, find the two successors of - // the parent block of this BR_IF. - auto *ParentMBB = MI.getParent(); - assert(ParentMBB->succ_size() == 2); + // the parent block of this BR_IF. (If there are three successors, due to some + // terminator before us, give up on this hint.) + if (ParentMBB->succ_size() != 2) + return {}; auto iter = ParentMBB->succ_begin(); MachineBasicBlock* MBB1 = *iter; iter++; @@ -706,6 +716,22 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) auto *FalseDest = &*ParentMBBI; // It must be one of the successors. assert(FalseDest == MBB1 || FalseDest == MBB2); + // Stop if the fallthrough is an EH pad, because that could happen like this: + // + // bb1: + // ;; successor: if.true, otherwise + // .. + // br_if $if.true + // br $otherwise + // + // ehpad: + // + // If that br $otherwise is optimized out, it would look like we fall through + // to the ehpad (since it is the block physically after us), but in wasm's + // structured control flow that is not the case. Rather than try to find the + // true fallthrough, give up on a hint in this rare case. + if (FalseDest->isEHPad()) + return {}; // The true destination (i.e. if the condition is true) is the other one. MachineBasicBlock *TrueDest = (FalseDest == MBB1 ? MBB2 : MBB1); From c492b0564434db10f7c67ee5cd9d4def3e9f3295 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 25 Jun 2025 10:01:09 -0700 Subject: [PATCH 53/61] Get MBB/Terminator code to compile --- llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index b65ae94859772..8b36b265d4f63 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -687,10 +687,10 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // We only handle the simple case of our being the last terminator of the // block. If there are other terminators after us, things may be complicated. auto *ParentMBB = MI.getParent(); - auto TerminatorRange = MBB.terminators(); - assert(!TerminatorRange.empty()); - auto *LastTerminator = &*TerminatorRange.rbegin(); - if (&MI != LastTerminator) + const MachineInstr *LastTerminator = nullptr; + for (auto& Terminator : ParentMBB->terminators()) + LastTerminator = &Terminator; + if (LastTerminator != &MI) return {}; // This is a BR. It has two successors, and perhaps branch probability From aee5b2ac83ed31dc8952467395c13c1d15bf0503 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 25 Jun 2025 13:40:25 -0700 Subject: [PATCH 54/61] almost get lld changes to compile --- lld/wasm/OutputSections.cpp | 38 +++++++++++++++++++------------------ lld/wasm/OutputSections.h | 2 ++ lld/wasm/Writer.cpp | 4 ++-- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/lld/wasm/OutputSections.cpp b/lld/wasm/OutputSections.cpp index 2b2f372f84526..ab51e563f16b2 100644 --- a/lld/wasm/OutputSections.cpp +++ b/lld/wasm/OutputSections.cpp @@ -284,7 +284,7 @@ void CustomSection::writeRelocations(raw_ostream &os) const { // Branch Hints -void BranchHintSection::BranchHintSection(ArrayRef inputSections) : CustomSection("metadata.code.branch_hint", inputSections) { +BranchHintSection::BranchHintSection(ArrayRef inputSections) : CustomSection(sectionName(), inputSections) { // Replace the original input sections with artificial ones. The change we // make is to add the total number of functions across all sections to the // first section's first 5 bytes, and then to delete the first 5 bytes in all @@ -297,51 +297,53 @@ void BranchHintSection::BranchHintSection(ArrayRef inputSections) } auto *newInputSections = make>(); - newInputSections.resize(inputSections.size()); + newInputSections->resize(inputSections.size()); // Remove the first 5 bytes from all sections but the first, and count how // many functions there are (so we can add that to the first). uint64_t totalFunctions = 0; for (unsigned i = 1; i < inputSections.size(); i++) { - const InputChunk *section = inputSections[i]; - assert(section.kind() == InputChunk::Section); + assert(InputSection::classof(inputSections[i])); + auto *section = static_cast(inputSections[i]); + const WasmSection &wasmSection = section->section; // Read the number of functions in this section. - totalFunctions += decodeULEB128(section->wasmSection.Content.data()); + totalFunctions += decodeULEB128(wasmSection.Content.data()); // Create an adjusted wasm section, without the first 5 bytes. - WasmSection *adjustedWasmSection = make(section->wasmSection); - adjustedWasmSection.Content = adjustedWasmSection.Content.slice(5); - for (auto& relocation : adjustedWasmSection->relocations) - relocation.offset -= 5; + WasmSection *adjustedWasmSection = make(wasmSection); + adjustedWasmSection->Content = adjustedWasmSection->Content.slice(5); + for (auto& relocation : adjustedWasmSection->Relocations) + relocation.Offset -= 5; - newInputSections[i] = make(adjustedWasmSection, section->file, section->alignment); + (*newInputSections)[i] = make(adjustedWasmSection, section->file, section->alignment); } // Add the number of functions to the first section. { - const InputChunk *section = inputSections[0]; - assert(section.kind() == InputChunk::Section); + assert(InputSection::classof(inputSections[0])); + auto *section = static_cast(inputSections[0]); + const WasmSection &wasmSection = section->section; // Read the number of functions in this section. - totalFunctions += decodeULEB128(section->wasmSection.Content.data()); + totalFunctions += decodeULEB128(wasmSection.Content.data()); // Create an adjusted wasm section, with the first 5 bytes modified so that // we apply the total number of functions. - WasmSection *adjustedWasmSection = make(section->wasmSection); - adjustedWasmSection.Content = adjustedWasmSection.Content.copy(); + WasmSection *adjustedWasmSection = make(wasmSection); + adjustedWasmSection->Content = adjustedWasmSection->Content.copy(getSpecificAllocSingleton()); std::string str; raw_string_ostream os(str); encodeULEB128(totalFunctions, os); // XXX? os << name; - memcpy(adjustedWasmSection.Content.data(), str.data(), 5); + memcpy(adjustedWasmSection->Content.data(), str.data(), 5); - newInputSections[i] = make(adjustedWasmSection, section->file, section->alignment); + (*newInputSections)[0] = make(adjustedWasmSection, section->file, section->alignment); } // Use these new artificial sections. - inputSections = newInputSections; + inputSections = ArrayRef(*newInputSections); } } // namespace wasm diff --git a/lld/wasm/OutputSections.h b/lld/wasm/OutputSections.h index 3d46ca6394a6a..e292d9bf8ac30 100644 --- a/lld/wasm/OutputSections.h +++ b/lld/wasm/OutputSections.h @@ -161,6 +161,8 @@ class CustomSection : public OutputSection { class BranchHintSection : public CustomSection { public: BranchHintSection(ArrayRef inputSections); + + static const char* sectionName() { return "metadata.code.branch_hint"; } }; } // namespace wasm diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp index 83138d86528ea..eceac4165c76a 100644 --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -182,7 +182,7 @@ void Writer::createCustomSections() { } void Writer::createBranchHintSection() { - StringRef name = "metadata.code.branch_hint"; + StringRef name = BranchHintSection::sectionName(); auto iter = customSectionMapping.find(name); if (iter == customSectionMapping.end()) return; @@ -190,7 +190,7 @@ void Writer::createBranchHintSection() { dbgs() << "createBranchHintSection!: " << name << "\n"; - OutputSection *sec = make(std::string(name), inputChunks); + auto *sec = make(inputChunks); auto *sym = make(sec); out.linkingSec->addToSymtab(sym); sec->sectionSym = sym; From 6b327913fb120649ec6263c91eda0608844b7a72 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 25 Jun 2025 16:07:44 -0700 Subject: [PATCH 55/61] lld compiles --- lld/wasm/OutputSections.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lld/wasm/OutputSections.cpp b/lld/wasm/OutputSections.cpp index ab51e563f16b2..d3d55a9d1aa3e 100644 --- a/lld/wasm/OutputSections.cpp +++ b/lld/wasm/OutputSections.cpp @@ -316,7 +316,7 @@ BranchHintSection::BranchHintSection(ArrayRef inputSections) : Cus for (auto& relocation : adjustedWasmSection->Relocations) relocation.Offset -= 5; - (*newInputSections)[i] = make(adjustedWasmSection, section->file, section->alignment); + (*newInputSections)[i] = make(*adjustedWasmSection, section->file, section->alignment); } // Add the number of functions to the first section. @@ -331,15 +331,16 @@ BranchHintSection::BranchHintSection(ArrayRef inputSections) : Cus // Create an adjusted wasm section, with the first 5 bytes modified so that // we apply the total number of functions. WasmSection *adjustedWasmSection = make(wasmSection); - adjustedWasmSection->Content = adjustedWasmSection->Content.copy(getSpecificAllocSingleton()); + auto* adjustedContent = make>(adjustedWasmSection->Content.begin(), adjustedWasmSection->Content.end()); std::string str; raw_string_ostream os(str); encodeULEB128(totalFunctions, os); // XXX? os << name; - memcpy(adjustedWasmSection->Content.data(), str.data(), 5); + memcpy(adjustedContent->data(), str.data(), 5); + adjustedWasmSection->Content = ArrayRef(adjustedContent->data(), adjustedContent->size()); - (*newInputSections)[0] = make(adjustedWasmSection, section->file, section->alignment); + (*newInputSections)[0] = make(*adjustedWasmSection, section->file, section->alignment); } // Use these new artificial sections. From e98fc218a4bc48c7e737a2e1cdd2c9fa5b428d77 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 25 Jun 2025 16:12:21 -0700 Subject: [PATCH 56/61] fix tests for padded num functions --- lld/test/wasm/branch-hints.ll | 4 ++-- llvm/test/CodeGen/WebAssembly/branch-hints.ll | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lld/test/wasm/branch-hints.ll b/lld/test/wasm/branch-hints.ll index 5e5d757df1311..49cd0ffe9abda 100644 --- a/lld/test/wasm/branch-hints.ll +++ b/lld/test/wasm/branch-hints.ll @@ -22,8 +22,8 @@ success: ; CHECK: - Type: CUSTOM ; CHECK-NEXT: Name: metadata.code.branch_hint -; CHECK-NEXT: Payload: '01818080800001080100' -; ^^ one function +; CHECK-NEXT: Payload: '8180808000818080800001080100' +; ^^ one function (5-byte padded LEB) ; ^^^^^^^^^^ LEB of function index 1 ; ^^ one hint in function ; ^^ offset 8 diff --git a/llvm/test/CodeGen/WebAssembly/branch-hints.ll b/llvm/test/CodeGen/WebAssembly/branch-hints.ll index 9a4eaacbea318..51fd390259238 100644 --- a/llvm/test/CodeGen/WebAssembly/branch-hints.ll +++ b/llvm/test/CodeGen/WebAssembly/branch-hints.ll @@ -17,8 +17,8 @@ entry: ; CHECK: .section .custom_section.metadata.code.branch_hint,"",@ -; Number of functions with hints. -; CHECK-NEXT: .int8 1 +; Number of functions with hints (1, padded LEB to 5 bytes +; CHECK-NEXT: .asciz "\201\200\200\200" ; Function with the hint. ; CHECK-NEXT: .uleb128_int32 bw_bh_test From 46650d98683813084723f8685783459c88bb617c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 25 Jun 2025 16:19:44 -0700 Subject: [PATCH 57/61] start new tests --- .../wasm/Inputs/branch-hints-multifile.ll | 14 +++++++++ lld/test/wasm/branch-hints-multifile.ll | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 lld/test/wasm/Inputs/branch-hints-multifile.ll create mode 100644 lld/test/wasm/branch-hints-multifile.ll diff --git a/lld/test/wasm/Inputs/branch-hints-multifile.ll b/lld/test/wasm/Inputs/branch-hints-multifile.ll new file mode 100644 index 0000000000000..ded4e546242a3 --- /dev/null +++ b/lld/test/wasm/Inputs/branch-hints-multifile.ll @@ -0,0 +1,14 @@ +define i32 @bw_bh_test_2(i32 %a, i32 %b) { +entry: + %1 = icmp ult i32 %a, %b + br i1 %1, label %fail, label %success, !prof !0 + +fail: + ret i32 -1 + +success: + ret i32 0 +} + +!0 = !{!"branch_weights", !"expected", i32 1, i32 2000} + diff --git a/lld/test/wasm/branch-hints-multifile.ll b/lld/test/wasm/branch-hints-multifile.ll new file mode 100644 index 0000000000000..63d4ec72fbd9b --- /dev/null +++ b/lld/test/wasm/branch-hints-multifile.ll @@ -0,0 +1,29 @@ + +; RUN: llc -mtriple=wasm32-unknown-unknown -filetype=obj -o %t1.o < %s +; RUN: llc -mtriple=wasm32-unknown-unknown -filetype=obj -o %t2.o < %S/Inputs/branch-hints-multifile.ll +; RUN: wasm-ld -o %t.wasm %t1.o %t2.o --no-entry --no-gc-sections +; RUN: obj2yaml %t.wasm | FileCheck %s + +define i32 @bw_bh_test_1(i32 %a, i32 %b) { +entry: + %1 = icmp ult i32 %a, %b + br i1 %1, label %fail, label %success, !prof !0 + +fail: + ret i32 -1 + +success: + ret i32 0 +} + +!0 = !{!"branch_weights", !"expected", i32 2000, i32 1} + +; CHECK: - Type: CUSTOM +; CHECK-NEXT: Name: metadata.code.branch_hint +; CHECK-NEXT: Payload: '8180808000818080800001080100' +; ^^ one function (5-byte padded LEB) +; ^^^^^^^^^^ LEB of function index 1 +; ^^ one hint in function +; ^^ offset 8 +; ^^ hint size 1 +; ^^ hint value: 0 From 0627b7cdf96e196b36c57249e931e853ed50ab19 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 26 Jun 2025 09:36:03 -0700 Subject: [PATCH 58/61] improve cfg analysis following feedback --- .../WebAssembly/WebAssemblyAsmPrinter.cpp | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 8b36b265d4f63..54bc11b2cbd2b 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -697,8 +697,8 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // info between them. Finding the successor blocks is not trivial, since we // run after CFGstackify (and even WebAssemblyInstrInfo::analyzeBranch returns // that it cannot analyze branch targets). First, find the two successors of - // the parent block of this BR_IF. (If there are three successors, due to some - // terminator before us, give up on this hint.) + // the parent block of this BR_IF. (If there are three successors, due to an + // additional unwind (EH pad) successor, give up on this hint.) if (ParentMBB->succ_size() != 2) return {}; auto iter = ParentMBB->succ_begin(); @@ -706,17 +706,15 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) iter++; MachineBasicBlock* MBB2 = *iter; - // See which of the two successors is right after us: the fallthrough. If we - // branch, it is to the other block. So the block right after us is where we - // go if the br_if condition is false. + // Find the fallthrough, that is, the block right after us, where control flow + // goes if we do not branch. auto ParentMBBI = ParentMBB->getIterator(); ++ParentMBBI; // A block with a BR_IF must have something after it. assert(ParentMBBI != MF->end()); - auto *FalseDest = &*ParentMBBI; - // It must be one of the successors. - assert(FalseDest == MBB1 || FalseDest == MBB2); - // Stop if the fallthrough is an EH pad, because that could happen like this: + auto *Fallthrough = &*ParentMBBI; + // In some corner cases, wasm's structured control flow makes it hard to infer + // control flow, like this: // // bb1: // ;; successor: if.true, otherwise @@ -726,13 +724,18 @@ std::optional WebAssemblyAsmPrinter::getBranchHint(const MachineInstr& MI) // // ehpad: // - // If that br $otherwise is optimized out, it would look like we fall through + // If `br $otherwise` is optimized out, it would look like we fall through // to the ehpad (since it is the block physically after us), but in wasm's // structured control flow that is not the case. Rather than try to find the - // true fallthrough, give up on a hint in this rare case. - if (FalseDest->isEHPad()) + // true fallthrough, give up on a hint in any case where the fallthrough is + // not one of our block's successors. + if (Fallthrough != MBB1 && Fallthrough != MBB2) return {}; - // The true destination (i.e. if the condition is true) is the other one. + // We ruled out complex cases, so what is left is simple control flow, and the + // false destination, i.e. where we go if the br_if condition is false, is the + // the fallthrough. + auto *FalseDest = Fallthrough; + // The true destination, i.e. if the condition is true, is the other one. MachineBasicBlock *TrueDest = (FalseDest == MBB1 ? MBB2 : MBB1); // Find the probability of branching. From 87f0a6819353f31f30e4a8407945a9f117134ac9 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 26 Jun 2025 09:46:11 -0700 Subject: [PATCH 59/61] DEBUG the bizarre problem --- lld/wasm/OutputSections.cpp | 33 ++++++++++++++++++++++++++------- lld/wasm/Writer.cpp | 3 +++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/lld/wasm/OutputSections.cpp b/lld/wasm/OutputSections.cpp index d3d55a9d1aa3e..860fa9553f278 100644 --- a/lld/wasm/OutputSections.cpp +++ b/lld/wasm/OutputSections.cpp @@ -265,6 +265,12 @@ void CustomSection::writeTo(uint8_t *buf) { memcpy(buf, nameData.data(), nameData.size()); buf += nameData.size(); +if (name == BranchHintSection::sectionName()) { + for (int i = 0; i < 5; i++) errs() << " Writing! [" << i << "] = " << int(static_cast(inputSections[0])->section.Content[i]) << '\n'; +} else { +errs() << "other sec: " << name << '\n'; +} + // Write custom sections payload for (const InputChunk *section : inputSections) section->writeTo(buf); @@ -290,14 +296,15 @@ BranchHintSection::BranchHintSection(ArrayRef inputSections) : Cus // first section's first 5 bytes, and then to delete the first 5 bytes in all // subsequent sections. After that, simply concatenating the sections leads to // the proper output. +errs() << "BANCH!\n"; assert(!inputSections.empty()); if (inputSections.size() == 1) { // Nothing to do. return; } +errs() << "BANCH2!\n"; - auto *newInputSections = make>(); - newInputSections->resize(inputSections.size()); + std::vector newInputSections(inputSections.size()); // Remove the first 5 bytes from all sections but the first, and count how // many functions there are (so we can add that to the first). @@ -316,9 +323,12 @@ BranchHintSection::BranchHintSection(ArrayRef inputSections) : Cus for (auto& relocation : adjustedWasmSection->Relocations) relocation.Offset -= 5; - (*newInputSections)[i] = make(*adjustedWasmSection, section->file, section->alignment); + newInputSections[i] = make(*adjustedWasmSection, section->file, section->alignment); } +errs() << "BANCH extras " << totalFunctions << "\n"; + + // Add the number of functions to the first section. { assert(InputSection::classof(inputSections[0])); @@ -328,23 +338,32 @@ BranchHintSection::BranchHintSection(ArrayRef inputSections) : Cus // Read the number of functions in this section. totalFunctions += decodeULEB128(wasmSection.Content.data()); +errs() << "BANCH final total..: " << totalFunctions << "\n"; + // Create an adjusted wasm section, with the first 5 bytes modified so that // we apply the total number of functions. WasmSection *adjustedWasmSection = make(wasmSection); auto* adjustedContent = make>(adjustedWasmSection->Content.begin(), adjustedWasmSection->Content.end()); +for (int i = 0; i < 5; i++) errs() << " adjustedContent[" << i << "] = " << int((*adjustedContent)[i]) << '\n'; std::string str; raw_string_ostream os(str); - encodeULEB128(totalFunctions, os); + encodeULEB128(totalFunctions, os, 5); +for (int i = 0; i < 5; i++) errs() << " str[" << i << "] = " << int(str[i]) << '\n'; // XXX? os << name; memcpy(adjustedContent->data(), str.data(), 5); +for (int i = 0; i < 5; i++) errs() << " adjustedContent[" << i << "] = " << int((*adjustedContent)[i]) << '\n'; adjustedWasmSection->Content = ArrayRef(adjustedContent->data(), adjustedContent->size()); - (*newInputSections)[0] = make(*adjustedWasmSection, section->file, section->alignment); + newInputSections[0] = make(*adjustedWasmSection, section->file, section->alignment); } - // Use these new artificial sections. - inputSections = ArrayRef(*newInputSections); +for (int i = 0; i < 5; i++) errs() << " old[" << i << "] = " << int(static_cast(inputSections[0])->section.Content[i]) << '\n'; + + // Use these new adjusted sections. + inputSections = std::move(newInputSections); + +for (int i = 0; i < 5; i++) errs() << " new[" << i << "] = " << int(static_cast(inputSections[0])->section.Content[i]) << '\n'; } } // namespace wasm diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp index eceac4165c76a..ddc8f043eef95 100644 --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -195,6 +195,7 @@ void Writer::createBranchHintSection() { out.linkingSec->addToSymtab(sym); sec->sectionSym = sym; addSection(sec); +errs() << "added.\n"; // After emitting this section, avoid processing it again in the place that // custom sections are normally created, later in the binary (inside @@ -244,11 +245,13 @@ void Writer::writeHeader() { } void Writer::writeSections() { +errs() << "writeSections1\n"; uint8_t *buf = buffer->getBufferStart(); parallelForEach(outputSections, [buf](OutputSection *s) { assert(s->isNeeded()); s->writeTo(buf); }); +errs() << "writeSections2\n"; } // Computes a hash value of Data using a given hash function. From 899ba0a19091fe6a4378ee0175360ec94fb7b02c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 26 Jun 2025 11:15:28 -0700 Subject: [PATCH 60/61] cleanup --- lld/wasm/OutputSections.cpp | 84 ---------------------------- lld/wasm/OutputSections.h | 33 ----------- lld/wasm/Writer.cpp | 108 +++++++++++++++++++++++++++++++++--- 3 files changed, 100 insertions(+), 125 deletions(-) diff --git a/lld/wasm/OutputSections.cpp b/lld/wasm/OutputSections.cpp index 860fa9553f278..8ccd38f7895cb 100644 --- a/lld/wasm/OutputSections.cpp +++ b/lld/wasm/OutputSections.cpp @@ -265,12 +265,6 @@ void CustomSection::writeTo(uint8_t *buf) { memcpy(buf, nameData.data(), nameData.size()); buf += nameData.size(); -if (name == BranchHintSection::sectionName()) { - for (int i = 0; i < 5; i++) errs() << " Writing! [" << i << "] = " << int(static_cast(inputSections[0])->section.Content[i]) << '\n'; -} else { -errs() << "other sec: " << name << '\n'; -} - // Write custom sections payload for (const InputChunk *section : inputSections) section->writeTo(buf); @@ -288,83 +282,5 @@ void CustomSection::writeRelocations(raw_ostream &os) const { s->writeRelocations(os); } -// Branch Hints - -BranchHintSection::BranchHintSection(ArrayRef inputSections) : CustomSection(sectionName(), inputSections) { - // Replace the original input sections with artificial ones. The change we - // make is to add the total number of functions across all sections to the - // first section's first 5 bytes, and then to delete the first 5 bytes in all - // subsequent sections. After that, simply concatenating the sections leads to - // the proper output. -errs() << "BANCH!\n"; - assert(!inputSections.empty()); - if (inputSections.size() == 1) { - // Nothing to do. - return; - } -errs() << "BANCH2!\n"; - - std::vector newInputSections(inputSections.size()); - - // Remove the first 5 bytes from all sections but the first, and count how - // many functions there are (so we can add that to the first). - uint64_t totalFunctions = 0; - for (unsigned i = 1; i < inputSections.size(); i++) { - assert(InputSection::classof(inputSections[i])); - auto *section = static_cast(inputSections[i]); - const WasmSection &wasmSection = section->section; - - // Read the number of functions in this section. - totalFunctions += decodeULEB128(wasmSection.Content.data()); - - // Create an adjusted wasm section, without the first 5 bytes. - WasmSection *adjustedWasmSection = make(wasmSection); - adjustedWasmSection->Content = adjustedWasmSection->Content.slice(5); - for (auto& relocation : adjustedWasmSection->Relocations) - relocation.Offset -= 5; - - newInputSections[i] = make(*adjustedWasmSection, section->file, section->alignment); - } - -errs() << "BANCH extras " << totalFunctions << "\n"; - - - // Add the number of functions to the first section. - { - assert(InputSection::classof(inputSections[0])); - auto *section = static_cast(inputSections[0]); - const WasmSection &wasmSection = section->section; - - // Read the number of functions in this section. - totalFunctions += decodeULEB128(wasmSection.Content.data()); - -errs() << "BANCH final total..: " << totalFunctions << "\n"; - - // Create an adjusted wasm section, with the first 5 bytes modified so that - // we apply the total number of functions. - WasmSection *adjustedWasmSection = make(wasmSection); - auto* adjustedContent = make>(adjustedWasmSection->Content.begin(), adjustedWasmSection->Content.end()); -for (int i = 0; i < 5; i++) errs() << " adjustedContent[" << i << "] = " << int((*adjustedContent)[i]) << '\n'; - - std::string str; - raw_string_ostream os(str); - encodeULEB128(totalFunctions, os, 5); -for (int i = 0; i < 5; i++) errs() << " str[" << i << "] = " << int(str[i]) << '\n'; - // XXX? os << name; - memcpy(adjustedContent->data(), str.data(), 5); -for (int i = 0; i < 5; i++) errs() << " adjustedContent[" << i << "] = " << int((*adjustedContent)[i]) << '\n'; - adjustedWasmSection->Content = ArrayRef(adjustedContent->data(), adjustedContent->size()); - - newInputSections[0] = make(*adjustedWasmSection, section->file, section->alignment); - } - -for (int i = 0; i < 5; i++) errs() << " old[" << i << "] = " << int(static_cast(inputSections[0])->section.Content[i]) << '\n'; - - // Use these new adjusted sections. - inputSections = std::move(newInputSections); - -for (int i = 0; i < 5; i++) errs() << " new[" << i << "] = " << int(static_cast(inputSections[0])->section.Content[i]) << '\n'; -} - } // namespace wasm } // namespace lld diff --git a/lld/wasm/OutputSections.h b/lld/wasm/OutputSections.h index e292d9bf8ac30..4b0329dd16cf2 100644 --- a/lld/wasm/OutputSections.h +++ b/lld/wasm/OutputSections.h @@ -132,39 +132,6 @@ class CustomSection : public OutputSection { std::string nameData; }; -// A Branch Hint section is a Custom Section with some custom rules for how it -// is created. Rather than simply concatenate the input sections, we must also -// adjust the field that reports the number of functions. -// -// Number of functions with hints. We pad this to 5 bytes to make the linker's -// life easier: given multiple Branch Hint sections, wasm-ld will by default -// simply concatenate them, just like any other custom section. That would end -// up with -// -// [num functions_1] : 5 byte LEB -// [..data_1..] -// [num functions_2] : 5 byte LEB -// [..data_2..] -// .. -// [num functions_N] : 5 byte LEB -// [..data_N..] -// -// To get this to the proper form the linker should emit we can trample the -// number of functions at the very start, and not emit the others: -// -// [num functions_1 + _2 + .. + _N] : 5 byte LEB -// [..data_1..] -// [..data_2..] -// .. -// [..data_N..] -// -class BranchHintSection : public CustomSection { -public: - BranchHintSection(ArrayRef inputSections); - - static const char* sectionName() { return "metadata.code.branch_hint"; } -}; - } // namespace wasm } // namespace lld diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp index ddc8f043eef95..443bcba2e2692 100644 --- a/lld/wasm/Writer.cpp +++ b/lld/wasm/Writer.cpp @@ -28,6 +28,7 @@ #include "llvm/BinaryFormat/Wasm.h" #include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/LEB128.h" #include "llvm/Support/Parallel.h" #include "llvm/Support/RandomNumberGenerator.h" #include "llvm/Support/SHA1.h" @@ -181,8 +182,42 @@ void Writer::createCustomSections() { } } +// A Branch Hint section is a Custom Section with some custom rules for how it +// is created. Rather than simply concatenate the input sections, we must also +// adjust the field that reports the number of functions, as follows. +// +// Our input chunks each begin with a 5-byte LEB of the number of functions. If +// we simply concatenated, we'd get this: +// +// ;; from object file 1 +// [num functions_1] : 5 byte LEB +// [..data_1..] +// ;; from object file 2 +// [num functions_2] : 5 byte LEB +// [..data_2..] +// .. +// ;; from object file N +// [num functions_N] : 5 byte LEB +// [..data_N..] +// +// But the final output should report the total number of functions at the very +// start. To fix that, we must accumulate the total number of functions and use +// that at the very start (which comes from the first object file), and we must +// remove the first 5 bytes of the others: +// +// [num functions_1 + _2 + .. + _N] : 5 byte LEB +// [..data_1..] +// [..data_2..] +// .. +// [..data_N..] +// +// That is now correct. +// +// Also, the Branch Hint section must appear *before* the code, so we call this +// earlier than for other custom sections. void Writer::createBranchHintSection() { - StringRef name = BranchHintSection::sectionName(); + std::string name = "metadata.code.branch_hint"; + auto iter = customSectionMapping.find(name); if (iter == customSectionMapping.end()) return; @@ -190,15 +225,75 @@ void Writer::createBranchHintSection() { dbgs() << "createBranchHintSection!: " << name << "\n"; - auto *sec = make(inputChunks); + assert(!inputChunks.empty()); + CustomSection *sec; + if (inputChunks.size() == 1) { + // Just use the originals. We don't need to do any work. + sec = make(name, inputChunks); + } else { + // We need to merge the input chunks in the special format that the spec + // expects, as explained above. To do so, create new input chunks with those + // minor modifications, and then the normal custom section behavior of + // concatenating the chunks will give us the right output. + auto *newInputChunksAlloc = make>(inputChunks.size()); + auto &newInputChunks = *newInputChunksAlloc; + + // Remove the first 5 bytes from all sections but the first, and count how + // many functions there are (so we can add that to the first). + uint64_t totalFunctions = 0; + for (unsigned i = 1; i < inputChunks.size(); i++) { + assert(InputSection::classof(inputChunks[i])); + auto *section = static_cast(inputChunks[i]); + const WasmSection &wasmSection = section->section; + + // Read the number of functions in this section. + totalFunctions += decodeULEB128(wasmSection.Content.data()); + + // Create an adjusted wasm section, without the first 5 bytes. + WasmSection *adjustedWasmSection = make(wasmSection); + adjustedWasmSection->Content = adjustedWasmSection->Content.slice(5); + for (auto& relocation : adjustedWasmSection->Relocations) + relocation.Offset -= 5; + + newInputChunks[i] = make(*adjustedWasmSection, section->file, section->alignment); + newInputChunks[i]->setRelocations(adjustedWasmSection->Relocations); + } + + // Add the number of functions to the first section. + { + assert(InputSection::classof(inputChunks[0])); + auto *section = static_cast(inputChunks[0]); + const WasmSection &wasmSection = section->section; + + // Read the number of functions in this section. + totalFunctions += decodeULEB128(wasmSection.Content.data()); + + // Create an adjusted wasm section, with the first 5 bytes modified so that + // we apply the total number of functions. + WasmSection *adjustedWasmSection = make(wasmSection); + auto* adjustedContent = make>(adjustedWasmSection->Content.begin(), adjustedWasmSection->Content.end()); + + std::string str; + raw_string_ostream os(str); + encodeULEB128(totalFunctions, os, 5); + memcpy(adjustedContent->data(), str.data(), 5); + adjustedWasmSection->Content = ArrayRef(adjustedContent->data(), adjustedContent->size()); + + newInputChunks[0] = make(*adjustedWasmSection, section->file, section->alignment); + newInputChunks[0]->setRelocations(adjustedWasmSection->Relocations); + } + + sec = make(name, newInputChunks); + } + + // Otherwise, add the section normally, like any custom section. auto *sym = make(sec); out.linkingSec->addToSymtab(sym); sec->sectionSym = sym; addSection(sec); -errs() << "added.\n"; // After emitting this section, avoid processing it again in the place that - // custom sections are normally created, later in the binary (inside + // custom sections are normally created, which is later in the binary (inside // createCustomSections). customSectionMapping.erase("branch_hint"); } @@ -245,13 +340,11 @@ void Writer::writeHeader() { } void Writer::writeSections() { -errs() << "writeSections1\n"; uint8_t *buf = buffer->getBufferStart(); parallelForEach(outputSections, [buf](OutputSection *s) { assert(s->isNeeded()); s->writeTo(buf); }); -errs() << "writeSections2\n"; } // Computes a hash value of Data using a given hash function. @@ -573,8 +666,7 @@ void Writer::addSections() { addSection(out.elemSec); addSection(out.dataCountSec); - // The Branch Hints section is a special custom section that must be emitted - // before the code section. + // The Branch Hints section must be emitted before the code section. createBranchHintSection(); addSection(make(out.functionSec->inputFunctions)); From c87bcd143d2ab1e2f06fa96a5861a5e39a485363 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 26 Jun 2025 11:21:26 -0700 Subject: [PATCH 61/61] multifile test passes --- lld/test/wasm/branch-hints-multifile.ll | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lld/test/wasm/branch-hints-multifile.ll b/lld/test/wasm/branch-hints-multifile.ll index 63d4ec72fbd9b..57627df59aee9 100644 --- a/lld/test/wasm/branch-hints-multifile.ll +++ b/lld/test/wasm/branch-hints-multifile.ll @@ -18,12 +18,13 @@ success: !0 = !{!"branch_weights", !"expected", i32 2000, i32 1} +; Test that we combine branch hint sections properly. The number of functions +; should be reported once at the start (even though it appears in each object +; file, and the hints for each object file should then be concatenated). ; CHECK: - Type: CUSTOM ; CHECK-NEXT: Name: metadata.code.branch_hint -; CHECK-NEXT: Payload: '8180808000818080800001080100' -; ^^ one function (5-byte padded LEB) -; ^^^^^^^^^^ LEB of function index 1 -; ^^ one hint in function -; ^^ offset 8 -; ^^ hint size 1 -; ^^ hint value: 0 +; CHECK-NEXT: Payload: '8280808000818080800001080100828080800001080101' +; ^^ two functions (5-byte padded LEB) +; ^^hint for func 1^ +; ^^hint for func 2^ +