Skip to content

Commit e4b0ccd

Browse files
committed
[LLVM][WebAssembly] Implement branch hinting proposal
This commit implements the WebAssembly branch hinting proposal, as detailed at https://webassembly.github.io/branch-hinting/metadata/code/binary.html. This proposal introduces a mechanism to convey branch likelihood information to the WebAssembly engine, allowing for more effective performance optimizations. The proposal specifies a new custom section named `metadata.code.branch_hint`. This section can contain a sequence of hints, where each hint is a single byte that applies to a corresponding `br_if` or `if` instruction. The hint values are: - `0x00` (`unlikely`): The branch is unlikely to be taken. - `0x01` (`likely`): The branch is likely to be taken. This implementation includes the following changes: - Addition of the "branch-hinting" feature (flag) - Collection of edge probabilities in CFGStackify pass - Outputting of `metadata.code.branch_hint` section in WebAssemblyAsmPrinter - Addition of the `WebAssembly::Specifier::S_DEBUG_REF` symbol ref specifier - Custom relaxation of leb128 fragments for storage of uleb128 encoded function indices and instruction offsets - Custom handling of code metadata sections in lld, required since the proposal requires code metadata sections to start with a combined count of function hints, followed by an ordered list of function hints. This change is purely an optimization and does not alter the semantics of WebAssembly programs.
1 parent 1949536 commit e4b0ccd

28 files changed

+803
-12
lines changed

clang/include/clang/Driver/Options.td

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5244,6 +5244,8 @@ def mharden_sls_EQ : Joined<["-"], "mharden-sls=">, Group<m_Group>,
52445244

52455245
def matomics : Flag<["-"], "matomics">, Group<m_wasm_Features_Group>;
52465246
def mno_atomics : Flag<["-"], "mno-atomics">, Group<m_wasm_Features_Group>;
5247+
def mbranch_hinting : Flag<["-"], "mbranch-hinting">, Group<m_wasm_Features_Group>;
5248+
def mno_branch_hinting : Flag<["-"], "mno-branch-hinting">, Group<m_wasm_Features_Group>;
52475249
def mbulk_memory : Flag<["-"], "mbulk-memory">, Group<m_wasm_Features_Group>;
52485250
def mno_bulk_memory : Flag<["-"], "mno-bulk-memory">, Group<m_wasm_Features_Group>;
52495251
def mbulk_memory_opt : Flag<["-"], "mbulk-memory-opt">, Group<m_wasm_Features_Group>;

clang/lib/Basic/Targets/WebAssembly.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ bool WebAssemblyTargetInfo::setABI(const std::string &Name) {
5353
bool WebAssemblyTargetInfo::hasFeature(StringRef Feature) const {
5454
return llvm::StringSwitch<bool>(Feature)
5555
.Case("atomics", HasAtomics)
56+
.Case("branch-hinting", HasBranchHinting)
5657
.Case("bulk-memory", HasBulkMemory)
5758
.Case("bulk-memory-opt", HasBulkMemoryOpt)
5859
.Case("call-indirect-overlong", HasCallIndirectOverlong)
@@ -86,6 +87,8 @@ void WebAssemblyTargetInfo::getTargetDefines(const LangOptions &Opts,
8687
defineCPUMacros(Builder, "wasm", /*Tuning=*/false);
8788
if (HasAtomics)
8889
Builder.defineMacro("__wasm_atomics__");
90+
if (HasBranchHinting)
91+
Builder.defineMacro("__wasm_branch_hinting__");
8992
if (HasBulkMemory)
9093
Builder.defineMacro("__wasm_bulk_memory__");
9194
if (HasBulkMemoryOpt)
@@ -188,6 +191,7 @@ bool WebAssemblyTargetInfo::initFeatureMap(
188191
auto addBleedingEdgeFeatures = [&]() {
189192
addGenericFeatures();
190193
Features["atomics"] = true;
194+
Features["branch-hinting"] = true;
191195
Features["exception-handling"] = true;
192196
Features["extended-const"] = true;
193197
Features["fp16"] = true;
@@ -218,6 +222,14 @@ bool WebAssemblyTargetInfo::handleTargetFeatures(
218222
HasAtomics = false;
219223
continue;
220224
}
225+
if (Feature == "+branch-hinting") {
226+
HasBranchHinting = true;
227+
continue;
228+
}
229+
if (Feature == "-branch-hinting") {
230+
HasBranchHinting = false;
231+
continue;
232+
}
221233
if (Feature == "+bulk-memory") {
222234
HasBulkMemory = true;
223235
continue;

clang/lib/Basic/Targets/WebAssembly.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyTargetInfo : public TargetInfo {
5858
} SIMDLevel = NoSIMD;
5959

6060
bool HasAtomics = false;
61+
bool HasBranchHinting = false;
6162
bool HasBulkMemory = false;
6263
bool HasBulkMemoryOpt = false;
6364
bool HasCallIndirectOverlong = false;
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# RUN: rm -rf %t; split-file %s %t
2+
; RUN: llc -mcpu=mvp -mtriple=wasm32-unknown-unknown -filetype=obj %t/f1.ll -o %t/f1.o -mattr=+branch-hinting
3+
; RUN: llc -mcpu=mvp -mtriple=wasm32-unknown-unknown -filetype=obj %t/f2.ll -o %t/f2.o -mattr=+branch-hinting
4+
; RUN: wasm-ld --export-all -o %t.wasm %t/f2.o %t/f1.o
5+
; RUN: obj2yaml %t.wasm | FileCheck --check-prefixes=CHECK %s
6+
7+
; RUN: llc -mcpu=mvp -mtriple=wasm32-unknown-unknown -filetype=obj %t/f1.ll -o %t/f1.o -mattr=-branch-hinting
8+
; RUN: llc -mcpu=mvp -mtriple=wasm32-unknown-unknown -filetype=obj %t/f2.ll -o %t/f2.o -mattr=-branch-hinting
9+
; RUN: wasm-ld --export-all -o %t.wasm %t/f2.o %t/f1.o
10+
; RUN: obj2yaml %t.wasm | FileCheck --check-prefixes=NCHECK %s
11+
12+
; CHECK: - Type: CUSTOM
13+
; CHECK: Name: metadata.code.branch_hint
14+
; CHECK-NEXT: Entries:
15+
; CHECK-NEXT: - FuncIdx: 1
16+
; CHECK-NEXT: Hints:
17+
; CHECK-NEXT: - Offset: 7
18+
; CHECK-NEXT: Size: 1
19+
; CHECK-NEXT: Data: UNLIKELY
20+
; CHECK-NEXT: - Offset: 14
21+
; CHECK-NEXT: Size: 1
22+
; CHECK-NEXT: Data: LIKELY
23+
; CHECK-NEXT: - FuncIdx: 2
24+
; CHECK-NEXT: Hints:
25+
; CHECK-NEXT: - Offset: 5
26+
; CHECK-NEXT: Size: 1
27+
; CHECK-NEXT: Data: LIKELY
28+
; CHECK-NEXT: - FuncIdx: 3
29+
; CHECK-NEXT: Hints:
30+
; CHECK-NEXT: - Offset: 5
31+
; CHECK-NEXT: Size: 1
32+
; CHECK-NEXT: Data: UNLIKELY
33+
; CHECK-NEXT: - FuncIdx: 4
34+
; CHECK-NEXT: Hints:
35+
; CHECK-NEXT: - Offset: 5
36+
; CHECK-NEXT: Size: 1
37+
; CHECK-NEXT: Data: LIKELY
38+
39+
; CHECK: - Type: CUSTOM
40+
; CHECK-NEXT: Name: name
41+
; CHECK-NEXT: FunctionNames:
42+
; CHECK-NEXT: - Index: 0
43+
; CHECK-NEXT: Name: __wasm_call_ctors
44+
; CHECK-NEXT: - Index: 1
45+
; CHECK-NEXT: Name: test0
46+
; CHECK-NEXT: - Index: 2
47+
; CHECK-NEXT: Name: test1
48+
; CHECK-NEXT: - Index: 3
49+
; CHECK-NEXT: Name: _start
50+
; CHECK-NEXT: - Index: 4
51+
; CHECK-NEXT: Name: test_func1
52+
53+
; CHECK: - Type: CUSTOM
54+
; CHECK: Name: target_features
55+
; CHECK-NEXT: Features:
56+
; CHECK-NEXT: - Prefix: USED
57+
; CHECK-NEXT: Name: branch-hinting
58+
59+
60+
; NCHECK-NOT: Name: metadata.code.branch_hint
61+
; NCHECK-NOT: Name: branch-hinting
62+
63+
#--- f1.ll
64+
define i32 @_start(i32 %a) {
65+
entry:
66+
%cmp = icmp eq i32 %a, 0
67+
br i1 %cmp, label %if.then, label %if.else, !prof !0
68+
if.then:
69+
ret i32 1
70+
if.else:
71+
ret i32 2
72+
}
73+
74+
define i32 @test_func1(i32 %a) {
75+
entry:
76+
%cmp = icmp eq i32 %a, 0
77+
br i1 %cmp, label %if.then, label %if.else, !prof !1
78+
if.then:
79+
ret i32 1
80+
if.else:
81+
ret i32 2
82+
}
83+
84+
!0 = !{!"branch_weights", i32 2000, i32 1}
85+
!1 = !{!"branch_weights", i32 1, i32 2000}
86+
87+
#--- f2.ll
88+
target triple = "wasm32-unknown-unknown"
89+
90+
define i32 @test0(i32 %a) {
91+
entry:
92+
%cmp0 = icmp eq i32 %a, 0
93+
br i1 %cmp0, label %if.then, label %ret1, !prof !0
94+
if.then:
95+
%cmp1 = icmp eq i32 %a, 1
96+
br i1 %cmp1, label %ret1, label %ret2, !prof !1
97+
ret1:
98+
ret i32 2
99+
ret2:
100+
ret i32 1
101+
}
102+
103+
define i32 @test1(i32 %a) {
104+
entry:
105+
%cmp = icmp eq i32 %a, 0
106+
br i1 %cmp, label %if.then, label %if.else, !prof !1
107+
if.then:
108+
ret i32 1
109+
if.else:
110+
ret i32 2
111+
}
112+
113+
; the resulting branch hint is actually reversed, since llvm-br is turned into br_unless, inverting branch probs
114+
!0 = !{!"branch_weights", i32 2000, i32 1}
115+
!1 = !{!"branch_weights", i32 1, i32 2000}

lld/wasm/OutputSections.cpp

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,62 @@ void CustomSection::writeTo(uint8_t *buf) {
270270
section->writeTo(buf);
271271
}
272272

273+
void CodeMetaDataSection::writeTo(uint8_t *buf) {
274+
log("writing " + toString(*this) + " offset=" + Twine(offset) +
275+
" size=" + Twine(getSize()) + " chunks=" + Twine(inputSections.size()));
276+
277+
assert(offset);
278+
buf += offset;
279+
280+
// Write section header
281+
memcpy(buf, header.data(), header.size());
282+
buf += header.size();
283+
memcpy(buf, nameData.data(), nameData.size());
284+
buf += nameData.size();
285+
286+
uint32_t TotalNumHints = 0;
287+
for (const InputChunk *section :
288+
make_range(inputSections.rbegin(), inputSections.rend())) {
289+
section->writeTo(buf);
290+
unsigned EncodingSize;
291+
uint32_t NumHints =
292+
decodeULEB128(buf + section->outSecOff, &EncodingSize, nullptr);
293+
if (EncodingSize != 5) {
294+
fatal("Unexpected encoding size for function hint vec size in " + name +
295+
": must be exactly 5 bytes.");
296+
}
297+
TotalNumHints += NumHints;
298+
}
299+
encodeULEB128(TotalNumHints, buf, 5);
300+
}
301+
302+
void CodeMetaDataSection::finalizeContents() {
303+
finalizeInputSections();
304+
305+
raw_string_ostream os(nameData);
306+
encodeULEB128(name.size(), os);
307+
os << name;
308+
309+
bool firstSection = true;
310+
for (InputChunk *section : inputSections) {
311+
assert(!section->discarded);
312+
payloadSize = alignTo(payloadSize, section->alignment);
313+
if (firstSection) {
314+
section->outSecOff = payloadSize;
315+
payloadSize += section->getSize();
316+
firstSection = false;
317+
} else {
318+
// adjust output offset so that each section write overwrites exactly the
319+
// subsequent section's function hint vector size (which deduplicates)
320+
section->outSecOff = payloadSize - 5;
321+
// payload size should not include the hint vector size, which is deduped
322+
payloadSize += section->getSize() - 5;
323+
}
324+
}
325+
326+
createHeader(payloadSize + nameData.size());
327+
}
328+
273329
uint32_t CustomSection::getNumRelocations() const {
274330
uint32_t count = 0;
275331
for (const InputChunk *inputSect : inputSections)

lld/wasm/OutputSections.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,15 @@ class CustomSection : public OutputSection {
132132
std::string nameData;
133133
};
134134

135+
class CodeMetaDataSection : public CustomSection {
136+
public:
137+
CodeMetaDataSection(std::string name, ArrayRef<InputChunk *> inputSections)
138+
: CustomSection(name, inputSections) {}
139+
140+
void writeTo(uint8_t *buf) override;
141+
void finalizeContents() override;
142+
};
143+
135144
} // namespace wasm
136145
} // namespace lld
137146

lld/wasm/SyntheticSections.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ class SyntheticSection : public OutputSection {
4343

4444
void writeTo(uint8_t *buf) override {
4545
assert(offset);
46-
log("writing " + toString(*this));
46+
log("writing " + toString(*this) + " offset=" + Twine(offset) +
47+
" size=" + Twine(getSize()));
4748
memcpy(buf + offset, header.data(), header.size());
4849
memcpy(buf + offset + header.size(), body.data(), body.size());
4950
}

lld/wasm/Writer.cpp

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ class Writer {
9494
void addSections();
9595

9696
void createCustomSections();
97+
// branch hint custom section must be placed before code section
98+
void createBranchHintSection();
9799
void createSyntheticSections();
98100
void createSyntheticSectionsPostLayout();
99101
void finalizeSections();
@@ -164,15 +166,34 @@ void Writer::createCustomSections() {
164166
log("createCustomSections");
165167
for (auto &pair : customSectionMapping) {
166168
StringRef name = pair.first;
169+
if (name == "metadata.code.branch_hint") {
170+
// Branch hint section is created separately.
171+
continue;
172+
}
167173
LLVM_DEBUG(dbgs() << "createCustomSection: " << name << "\n");
174+
OutputSection *Sec = make<CustomSection>(std::string(name), pair.second);
175+
if (ctx.arg.relocatable || ctx.arg.emitRelocs) {
176+
auto *sym = make<OutputSectionSymbol>(Sec);
177+
out.linkingSec->addToSymtab(sym);
178+
Sec->sectionSym = sym;
179+
}
180+
addSection(Sec);
181+
}
182+
}
168183

169-
OutputSection *sec = make<CustomSection>(std::string(name), pair.second);
184+
void Writer::createBranchHintSection() {
185+
log("createBranchHintSection");
186+
std::string SectionName = "metadata.code.branch_hint";
187+
if (const auto &Ins = customSectionMapping.find(SectionName);
188+
Ins != customSectionMapping.end()) {
189+
OutputSection *Sec =
190+
make<CodeMetaDataSection>(std::move(SectionName), Ins->second);
170191
if (ctx.arg.relocatable || ctx.arg.emitRelocs) {
171-
auto *sym = make<OutputSectionSymbol>(sec);
192+
auto *sym = make<OutputSectionSymbol>(Sec);
172193
out.linkingSec->addToSymtab(sym);
173-
sec->sectionSym = sym;
194+
Sec->sectionSym = sym;
174195
}
175-
addSection(sec);
196+
addSection(Sec);
176197
}
177198
}
178199

@@ -544,6 +565,8 @@ void Writer::addSections() {
544565
addSection(out.elemSec);
545566
addSection(out.dataCountSec);
546567

568+
createBranchHintSection();
569+
547570
addSection(make<CodeSection>(out.functionSec->inputFunctions));
548571
addSection(make<DataSection>(segments));
549572

llvm/include/llvm/BinaryFormat/Wasm.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -517,6 +517,22 @@ struct WasmSignature {
517517
WasmSignature() = default;
518518
};
519519

520+
template <typename T> struct WasmCodeMetadataItemEntry {
521+
uint32_t Offset;
522+
uint32_t Size;
523+
T Data;
524+
};
525+
526+
template <typename T> struct WasmCodeMetadataFuncEntry {
527+
uint32_t FuncIdx;
528+
std::vector<WasmCodeMetadataItemEntry<T>> Hints;
529+
};
530+
531+
enum class WasmCodeMetadataBranchHint : uint8_t {
532+
UNLIKELY = 0x0,
533+
LIKELY = 0x1,
534+
};
535+
520536
// Useful comparison operators
521537
inline bool operator==(const WasmSignature &LHS, const WasmSignature &RHS) {
522538
return LHS.State == RHS.State && LHS.Returns == RHS.Returns &&

llvm/include/llvm/Object/Wasm.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,10 @@ class LLVM_ABI WasmObjectFile : public ObjectFile {
142142
ArrayRef<wasm::WasmFeatureEntry> getTargetFeatures() const {
143143
return TargetFeatures;
144144
}
145+
ArrayRef<wasm::WasmCodeMetadataFuncEntry<wasm::WasmCodeMetadataBranchHint>>
146+
getBranchHints() const {
147+
return BranchHints;
148+
}
145149
ArrayRef<wasm::WasmSignature> types() const { return Signatures; }
146150
ArrayRef<wasm::WasmImport> imports() const { return Imports; }
147151
ArrayRef<wasm::WasmTable> tables() const { return Tables; }
@@ -275,12 +279,16 @@ class LLVM_ABI WasmObjectFile : public ObjectFile {
275279
Error parseProducersSection(ReadContext &Ctx);
276280
Error parseTargetFeaturesSection(ReadContext &Ctx);
277281
Error parseRelocSection(StringRef Name, ReadContext &Ctx);
282+
Error parseCodeMetadataSection(StringRef Name, ReadContext &Ctx);
283+
Error parseBranchHintSection(ReadContext &Ctx);
278284

279285
wasm::WasmObjectHeader Header;
280286
std::vector<WasmSection> Sections;
281287
wasm::WasmDylinkInfo DylinkInfo;
282288
wasm::WasmProducerInfo ProducerInfo;
283289
std::vector<wasm::WasmFeatureEntry> TargetFeatures;
290+
std::vector<wasm::WasmCodeMetadataFuncEntry<wasm::WasmCodeMetadataBranchHint>>
291+
BranchHints;
284292
std::vector<wasm::WasmSignature> Signatures;
285293
std::vector<wasm::WasmTable> Tables;
286294
std::vector<wasm::WasmLimits> Memories;

0 commit comments

Comments
 (0)