Skip to content

Commit 3ba679b

Browse files
committed
[DTLTO][LLD][ELF] Add support for Integrated Distributed ThinLTO
This patch introduces support for Integrated Distributed ThinLTO (DTLTO) in ELF LLD. DTLTO enables the distribution of ThinLTO backend compilations via external distribution systems, such as Incredibuild, during the traditional link step: https://llvm.org/docs/DTLTO.html. It is expected that users will invoke DTLTO through the compiler driver (e.g., Clang) rather than calling LLD directly. A Clang-side interface for DTLTO will be added in a follow-up patch. Note: Bitcode members of non-thin archives are not currently supported. This will be addressed in a future change. Testing: - ELF LLD `lit` test coverage has been added, using a mock distributor to avoid requiring Clang. - Cross-project `lit` tests cover integration with Clang. For the design discussion of the DTLTO feature, see: #126654
1 parent 389e9d3 commit 3ba679b

20 files changed

+557
-8
lines changed

cross-project-tests/CMakeLists.txt

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,12 @@ set(CROSS_PROJECT_TEST_DEPS
1919
FileCheck
2020
check-gdb-llvm-support
2121
count
22-
llvm-dwarfdump
22+
llvm-ar
2323
llvm-config
24+
llvm-dwarfdump
2425
llvm-objdump
25-
split-file
2626
not
27+
split-file
2728
)
2829

2930
if ("clang" IN_LIST LLVM_ENABLE_PROJECTS)
@@ -94,6 +95,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
9495
DEPENDS clang
9596
)
9697

98+
# DTLTO tests.
99+
add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
100+
${CMAKE_CURRENT_BINARY_DIR}/dtlto
101+
EXCLUDE_FROM_CHECK_ALL
102+
DEPENDS ${CROSS_PROJECT_TEST_DEPS}
103+
)
104+
97105
# Add check-cross-project-* targets.
98106
add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
99107
DEPENDS ${CROSS_PROJECT_TEST_DEPS}

cross-project-tests/dtlto/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Tests for DTLTO (Integrated Distributed ThinLTO) functionality.
2+
3+
These are integration tests as DTLTO invokes `clang` for code-generation.
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# REQUIRES: x86-registered-target,ld.lld,llvm-ar
2+
3+
# Test that a DTLTO link succeeds and outputs the expected set of files
4+
# correctly when thin archives are present.
5+
6+
RUN: rm -rf %t.dir && split-file %s %t.dir && cd %t.dir
7+
RUN: %clang --target=x86_64-linux-gnu -c foo.c -o foo.o
8+
RUN: %clang --target=x86_64-linux-gnu -c -flto=thin bar.c -o bar.o
9+
RUN: %clang --target=x86_64-linux-gnu -c -flto=thin dog.c -o dog.o
10+
RUN: %clang --target=x86_64-linux-gnu -c -flto=thin cat.c -o cat.o
11+
RUN: %clang --target=x86_64-linux-gnu -c -flto=thin _start.c -o _start.o
12+
13+
RUN: llvm-ar rcs foo.a foo.o --thin
14+
# Create this bitcode thin archive in a sub-directory to test the expansion of
15+
# the path to a bitcode file which is referenced using "..", e.g. in this case
16+
# "../bar.o". The ".." should be collapsed in any expansion to avoid
17+
# referencing an unknown directory on the remote side.
18+
RUN: mkdir lib
19+
RUN: llvm-ar rcs lib/bar.a bar.o --thin
20+
# Create this bitcode thin archive with an absolute path entry containing "..".
21+
RUN: llvm-ar rcs dog.a %t.dir/lib/../dog.o --thin
22+
RUN: llvm-ar rcs cat.a cat.o --thin
23+
RUN: llvm-ar rcs _start.a _start.o --thin
24+
25+
RUN: mkdir %t.dir/out && cd %t.dir/out
26+
27+
RUN: ld.lld %t.dir/foo.a %t.dir/lib/bar.a ../_start.a %t.dir/cat.a \
28+
RUN: --whole-archive ../dog.a \
29+
RUN: --thinlto-distributor=%python \
30+
RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
31+
RUN: --thinlto-remote-compiler=%clang \
32+
RUN: --save-temps
33+
34+
# Check that the required output files have been created.
35+
RUN: ls | FileCheck %s --check-prefix=OUTPUTS \
36+
RUN: --implicit-check-not=cat --implicit-check-not=foo
37+
38+
# The DTLTO backend emits the JSON jobs description and summary shards.
39+
OUTPUTS-DAG: a.{{[0-9]+}}.dist-file.json
40+
OUTPUTS-DAG: bar.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}}
41+
OUTPUTS-DAG: dog.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}}
42+
OUTPUTS-DAG: _start.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}}
43+
# Native output object files.
44+
OUTPUTS-DAG: bar.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}}
45+
OUTPUTS-DAG: dog.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}}
46+
OUTPUTS-DAG: _start.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}}
47+
48+
# Check that bar.o and dog.o are not referenced using "..".
49+
RUN: not grep '\.\.\(/\|\\\\\)\(bar\|dog\)\.o' a.*.dist-file.json
50+
51+
#--- foo.c
52+
__attribute__((retain)) void foo() {}
53+
54+
#--- bar.c
55+
extern void foo();
56+
__attribute__((retain)) void bar() { foo(); }
57+
58+
#--- dog.c
59+
__attribute__((retain)) void dog() {}
60+
61+
#--- cat.c
62+
__attribute__((retain)) void cat() {}
63+
64+
#--- _start.c
65+
extern void bar();
66+
__attribute__((retain)) void _start() {
67+
bar();
68+
}

cross-project-tests/dtlto/dtlto.c

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// REQUIRES: x86-registered-target,ld.lld
2+
3+
/// Simple test that DTLTO works with a single input bitcode file and that
4+
/// --save-temps can be applied to the remote compilation.
5+
// RUN: rm -rf %t && mkdir %t && cd %t
6+
7+
// RUN: %clang --target=x86_64-linux-gnu -c %s -o dtlto.bc -flto=thin
8+
9+
// RUN: ld.lld %t/dtlto.bc \
10+
// RUN: --thinlto-distributor=%python \
11+
// RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
12+
// RUN: --thinlto-remote-compiler=%clang \
13+
// RUN: --thinlto-remote-compiler-arg=--save-temps
14+
15+
/// Check that the required output files have been created.
16+
// RUN: ls | count 10
17+
// RUN: ls | FileCheck %s
18+
19+
/// Produced by the bitcode compilation.
20+
// CHECK-DAG: {{^}}dtlto.bc{{$}}
21+
22+
/// Linked ELF.
23+
// CHECK-DAG: {{^}}a.out{{$}}
24+
25+
/// --save-temps output for the backend compilation.
26+
// CHECK-DAG: {{^}}dtlto.s{{$}}
27+
// CHECK-DAG: {{^}}dtlto.s.0.preopt.bc{{$}}
28+
// CHECK-DAG: {{^}}dtlto.s.1.promote.bc{{$}}
29+
// CHECK-DAG: {{^}}dtlto.s.2.internalize.bc{{$}}
30+
// CHECK-DAG: {{^}}dtlto.s.3.import.bc{{$}}
31+
// CHECK-DAG: {{^}}dtlto.s.4.opt.bc{{$}}
32+
// CHECK-DAG: {{^}}dtlto.s.5.precodegen.bc{{$}}
33+
// CHECK-DAG: {{^}}dtlto.s.resolution.txt{{$}}
34+
35+
int _start() { return 0; }
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
if "clang" not in config.available_features:
2+
config.unsupported = True

cross-project-tests/lit.cfg.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
2020

2121
# suffixes: A list of file extensions to treat as test files.
22-
config.suffixes = [".c", ".cl", ".cpp", ".m"]
22+
config.suffixes = [".c", ".cl", ".cpp", ".m", ".test"]
2323

2424
# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
2525
# subdirectories contain auxiliary inputs for various tests in their parent
@@ -107,6 +107,8 @@ def get_required_attr(config, attr_name):
107107
if lldb_path is not None:
108108
config.available_features.add("lldb")
109109

110+
if llvm_config.use_llvm_tool("llvm-ar"):
111+
config.available_features.add("llvm-ar")
110112

111113
def configure_dexter_substitutions():
112114
"""Configure substitutions for host platform and return list of dependencies"""

lld/ELF/Config.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,10 @@ struct Config {
249249
llvm::SmallVector<llvm::StringRef, 0> searchPaths;
250250
llvm::SmallVector<llvm::StringRef, 0> symbolOrderingFile;
251251
llvm::SmallVector<llvm::StringRef, 0> thinLTOModulesToCompile;
252+
llvm::StringRef dtltoDistributor;
253+
llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
254+
llvm::StringRef dtltoCompiler;
255+
llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
252256
llvm::SmallVector<llvm::StringRef, 0> undefined;
253257
llvm::SmallVector<SymbolVersion, 0> dynamicList;
254258
llvm::SmallVector<uint8_t, 0> buildIdVector;

lld/ELF/Driver.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1341,6 +1341,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
13411341
args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
13421342
ctx.arg.disableVerify = args.hasArg(OPT_disable_verify);
13431343
ctx.arg.discard = getDiscard(args);
1344+
ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq);
1345+
ctx.arg.dtltoDistributorArgs =
1346+
args::getStrings(args, OPT_thinlto_distributor_args);
1347+
ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler_eq);
1348+
ctx.arg.dtltoCompilerArgs = args::getStrings(args, OPT_thinlto_compiler_args);
13441349
ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
13451350
ctx.arg.dynamicLinker = getDynamicLinker(ctx, args);
13461351
ctx.arg.ehFrameHdr =

lld/ELF/InputFiles.cpp

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "llvm/ADT/CachedHashString.h"
2121
#include "llvm/ADT/STLExtras.h"
2222
#include "llvm/LTO/LTO.h"
23+
#include "llvm/Object/Archive.h"
2324
#include "llvm/Object/IRObjectFile.h"
2425
#include "llvm/Support/ARMAttributeParser.h"
2526
#include "llvm/Support/ARMBuildAttributes.h"
@@ -1739,6 +1740,40 @@ static uint8_t getOsAbi(const Triple &t) {
17391740
}
17401741
}
17411742

1743+
// For DTLTO, bitcode member names must be a valid path to a bitcode file on
1744+
// disk. For thin archives, adjust `memberPath` to the full file path of the
1745+
// archive member. Returns true if an adjustment was made; false otherwise.
1746+
// Non-thin archives are not yet supported.
1747+
static bool dtltoAdjustMemberPathIfThinArchive(Ctx &ctx, StringRef archivePath,
1748+
std::string &memberPath) {
1749+
assert(!archivePath.empty());
1750+
assert(!ctx.arg.dtltoDistributor.empty());
1751+
1752+
// Check if the archive file is a thin archive by reading its header.
1753+
auto memBufferOrError =
1754+
MemoryBuffer::getFileSlice(archivePath, sizeof(ThinArchiveMagic) - 1, 0);
1755+
if (std::error_code ec = memBufferOrError.getError()) {
1756+
ErrAlways(ctx) << "cannot open " << archivePath << ": " << ec.message();
1757+
return false;
1758+
}
1759+
MemoryBufferRef memBufRef = *memBufferOrError.get();
1760+
if (!memBufRef.getBuffer().starts_with(ThinArchiveMagic))
1761+
return false;
1762+
1763+
SmallString<64> fullMemberPath;
1764+
if (path::is_relative(memberPath)) {
1765+
fullMemberPath = path::parent_path(archivePath);
1766+
path::append(fullMemberPath, memberPath);
1767+
} else
1768+
fullMemberPath = memberPath;
1769+
1770+
//".." are collapsed to avoid referencing an unknown directory on the remote
1771+
// side.
1772+
path::remove_dots(fullMemberPath, /*remove_dot_dot=*/true);
1773+
memberPath = fullMemberPath.str();
1774+
return true;
1775+
}
1776+
17421777
BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName,
17431778
uint64_t offsetInArchive, bool lazy)
17441779
: InputFile(ctx, BitcodeKind, mb) {
@@ -1756,10 +1791,13 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName,
17561791
// symbols later in the link stage). So we append file offset to make
17571792
// filename unique.
17581793
StringSaver &ss = ctx.saver;
1759-
StringRef name = archiveName.empty()
1760-
? ss.save(path)
1761-
: ss.save(archiveName + "(" + path::filename(path) +
1762-
" at " + utostr(offsetInArchive) + ")");
1794+
StringRef name =
1795+
(archiveName.empty() ||
1796+
(!ctx.arg.dtltoDistributor.empty() &&
1797+
dtltoAdjustMemberPathIfThinArchive(ctx, archiveName, path)))
1798+
? ss.save(path)
1799+
: ss.save(archiveName + "(" + path::filename(path) + " at " +
1800+
utostr(offsetInArchive) + ")");
17631801
MemoryBufferRef mbref(mb.getBuffer(), name);
17641802

17651803
obj = CHECK2(lto::InputFile::create(mbref), this);

lld/ELF/LTO.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,14 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
180180
std::string(ctx.arg.thinLTOPrefixReplaceNew),
181181
std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
182182
ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
183+
} else if (!ctx.arg.dtltoDistributor.empty() && !ctx.bitcodeFiles.empty()) {
184+
backend = lto::createOutOfProcessThinBackend(
185+
llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),
186+
onIndexWrite, ctx.arg.thinLTOEmitIndexFiles,
187+
ctx.arg.thinLTOEmitImportsFiles, ctx.arg.outputFile,
188+
ctx.arg.dtltoDistributor, ctx.arg.dtltoDistributorArgs,
189+
ctx.arg.dtltoCompiler, ctx.arg.dtltoCompilerArgs,
190+
!ctx.arg.saveTempsArgs.empty());
183191
} else {
184192
backend = lto::createInProcessThinBackend(
185193
llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),

0 commit comments

Comments
 (0)