[DTLTO][LLD][ELF] Add support for Integrated Distributed ThinLTO

bd1976bris · bd1976bris · commit b8948245a9d6 · 2025-06-18T16:40:04.000+01:00
This patch introduces support for Integrated Distributed ThinLTO (DTLTO) in ELF LLD. DTLTO enables the distribution of ThinLTO backend compilations via external distribution systems, such as Incredibuild, during the traditional link step: https://llvm.org/docs/DTLTO.html. It is expected that users will invoke DTLTO through the compiler driver (e.g., Clang) rather than calling LLD directly. A Clang-side interface for DTLTO will be added in a follow-up patch. Note: Bitcode members of archives (both thin and non-thin) are not currently supported. This will be addressed in a future change. Testing: - ELF LLD `lit` test coverage has been added, using a mock distributor to avoid requiring Clang. - Cross-project `lit` tests cover integration with Clang. For the design discussion of the DTLTO feature, see: llvm#126654
diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt
@@ -19,11 +19,12 @@ set(CROSS_PROJECT_TEST_DEPS
   FileCheck
   check-gdb-llvm-support
   count
-  llvm-dwarfdump
+  llvm-ar
   llvm-config
+  llvm-dwarfdump
   llvm-objdump
-  split-file
   not
+  split-file
   )
 
 if ("clang" IN_LIST LLVM_ENABLE_PROJECTS)
@@ -94,6 +95,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
   DEPENDS clang
   )
 
+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${CROSS_PROJECT_TEST_DEPS}
diff --git a/cross-project-tests/dtlto/README.md b/cross-project-tests/dtlto/README.md
@@ -0,0 +1,3 @@
+Tests for DTLTO (Integrated Distributed ThinLTO) functionality.
+
+These are integration tests as DTLTO invokes `clang` for code-generation.
diff --git a/cross-project-tests/dtlto/dtlto.c b/cross-project-tests/dtlto/dtlto.c
@@ -0,0 +1,35 @@
+// REQUIRES: x86-registered-target,ld.lld
+
+/// Simple test that DTLTO works with a single input bitcode file and that
+/// --save-temps can be applied to the remote compilation.
+// RUN: rm -rf %t && mkdir %t && cd %t
+
+// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s
+
+// RUN: ld.lld dtlto.o \
+// RUN:   --thinlto-distributor=%python \
+// RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+// RUN:   --thinlto-remote-compiler=%clang \
+// RUN:   --thinlto-remote-compiler-arg=--save-temps
+
+/// Check that the required output files have been created.
+// RUN: ls | count 10
+// RUN: ls | FileCheck %s
+
+/// Produced by the bitcode compilation.
+// CHECK-DAG: {{^}}dtlto.o{{$}}
+
+/// Linked ELF.
+// CHECK-DAG: {{^}}a.out{{$}}
+
+/// --save-temps output for the backend compilation.
+// CHECK-DAG: {{^}}dtlto.s{{$}}
+// CHECK-DAG: {{^}}dtlto.s.0.preopt.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.1.promote.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.2.internalize.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.3.import.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.4.opt.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.5.precodegen.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.resolution.txt{{$}}
+
+int _start() { return 0; }
diff --git a/cross-project-tests/dtlto/lit.local.cfg b/cross-project-tests/dtlto/lit.local.cfg
@@ -0,0 +1,2 @@
+if "clang" not in config.available_features:
+    config.unsupported = True
diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py
@@ -19,7 +19,7 @@
 config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files.
-config.suffixes = [".c", ".cl", ".cpp", ".m"]
+config.suffixes = [".c", ".cl", ".cpp", ".m", ".test"]
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
@@ -249,6 +249,10 @@ struct Config {
   llvm::SmallVector<llvm::StringRef, 0> searchPaths;
   llvm::SmallVector<llvm::StringRef, 0> symbolOrderingFile;
   llvm::SmallVector<llvm::StringRef, 0> thinLTOModulesToCompile;
+  llvm::StringRef dtltoDistributor;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
+  llvm::StringRef dtltoCompiler;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
   llvm::SmallVector<llvm::StringRef, 0> undefined;
   llvm::SmallVector<SymbolVersion, 0> dynamicList;
   llvm::SmallVector<uint8_t, 0> buildIdVector;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
@@ -1341,6 +1341,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
   ctx.arg.disableVerify = args.hasArg(OPT_disable_verify);
   ctx.arg.discard = getDiscard(args);
+  ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq);
+  ctx.arg.dtltoDistributorArgs =
+      args::getStrings(args, OPT_thinlto_distributor_arg);
+  ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler_eq);
+  ctx.arg.dtltoCompilerArgs = args::getStrings(args, OPT_thinlto_compiler_arg);
   ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
   ctx.arg.dynamicLinker = getDynamicLinker(ctx, args);
   ctx.arg.ehFrameHdr =
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
@@ -20,6 +20,7 @@
 #include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/LTO/LTO.h"
+#include "llvm/Object/Archive.h"
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Support/ARMAttributeParser.h"
 #include "llvm/Support/ARMBuildAttributes.h"
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
@@ -180,6 +180,13 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
         std::string(ctx.arg.thinLTOPrefixReplaceNew),
         std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
         ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
+  } else if (!ctx.arg.dtltoDistributor.empty()) {
+    backend = lto::createOutOfProcessThinBackend(
+        llvm::hardware_concurrency(ctx.arg.thinLTOJobs), onIndexWrite,
+        ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles,
+        ctx.arg.outputFile, ctx.arg.dtltoDistributor,
+        ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler,
+        ctx.arg.dtltoCompilerArgs, !ctx.arg.saveTempsArgs.empty());
   } else {
     backend = lto::createInProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
@@ -710,7 +710,17 @@ def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
 def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
 def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
   HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">;
-
+def thinlto_distributor_eq: JJ<"thinlto-distributor=">,
+  HelpText<"Distributor to use for ThinLTO backend compilations. If specified, "
+  "ThinLTO backend compilations will be distributed.">;
+defm thinlto_distributor_arg: EEq<"thinlto-distributor-arg", "Arguments to "
+  "pass to the ThinLTO distributor">;
+def thinlto_compiler_eq: JJ<"thinlto-remote-compiler=">,
+  HelpText<"Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
+  "compilations">;
+defm thinlto_compiler_arg: EEq<"thinlto-remote-compiler-arg", "Compiler "
+  "arguments for the ThinLTO distributor to pass for ThinLTO backend "
+  "compilations">;
 defm fat_lto_objects: BB<"fat-lto-objects",
     "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.",
     "Ignore the .llvm.lto section in relocatable object files (default).">;
diff --git a/lld/docs/DTLTO.rst b/lld/docs/DTLTO.rst
@@ -0,0 +1,42 @@
+Integrated Distributed ThinLTO (DTLTO)
+======================================
+
+Integrated Distributed ThinLTO (DTLTO) enables the distribution of backend
+ThinLTO compilations via external distribution systems, such as Incredibuild,
+during the traditional link step.
+
+The implementation is documented here: https://llvm.org/docs/DTLTO.html.
+
+Currently, DTLTO is only supported in ELF LLD. Support will be added to other
+LLD flavours in the future.
+
+ELF LLD
+-------
+
+The command-line interface is as follows:
+
+- ``--thinlto-distributor=<path>``  
+  Specifies the file to execute as the distributor process. If specified,
+  ThinLTO backend compilations will be distributed.
+
+- ``--thinlto-remote-compiler=<path>``  
+  Specifies the path to the compiler that the distributor process will use for
+  backend compilations. The compiler invoked must match the version of LLD.
+
+- ``--thinlto-distributor-arg=<arg>``  
+  Specifies ``<arg>`` on the command line when invoking the distributor.
+  Can be specified multiple times.
+
+- ``--thinlto-remote-compiler-arg=<arg>``  
+  Appends ``<arg>`` to the remote compiler's command line.
+  Can be specified multiple times.
+
+  Options that introduce extra input/output files may cause miscompilation if
+  the distribution system does not automatically handle pushing/fetching them to
+  remote nodes. In such cases, configure the distributor - possibly using
+  ``--thinlto-distributor-arg=`` - to manage these dependencies. See the
+  distributor documentation for details.
+
+Some LLD LTO options (e.g., ``--lto-sample-profile=<file>``) are supported.
+Currently, other options are silently accepted but do not have the intended
+effect. Support for such options will be expanded in the future.
diff --git a/lld/docs/index.rst b/lld/docs/index.rst
@@ -147,3 +147,4 @@ document soon.
    ELF/start-stop-gc
    ELF/warn_backrefs
    MachO/index
+   DTLTO
diff --git a/lld/test/ELF/dtlto/empty.test b/lld/test/ELF/dtlto/empty.test
@@ -0,0 +1,44 @@
+# REQUIRES: x86
+
+# Test that DTLTO options are a no-op if no thinLTO is performed.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: opt t1.ll -o t1.o -O2
+RUN: opt t2.ll -o t2.o -O2
+
+# Common command-line arguments. Note that mock.py does not do any compilation;
+# instead, it simply writes the contents of the object files supplied on the
+# command line into the output object files in job order.
+RUN: echo "t1.o t2.o \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=no-exist1.o \
+RUN:   --thinlto-distributor-arg=no-exist2.o" > l.rsp
+
+# Check linking succeeds when all input files are Full LTO.
+RUN: ld.lld @l.rsp
+
+RUN: llc -filetype=obj t1.ll -o t1.o
+RUN: llc -filetype=obj t2.ll -o t2.o
+
+# Check linking succeeds when all input files are ET_RELs.
+RUN: ld.lld @l.rsp
+
+#--- t1.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @t1() {
+entry:
+  ret void
+}
+
+#--- t2.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @t2() {
+entry:
+  ret void
+}
diff --git a/lld/test/ELF/dtlto/imports.test b/lld/test/ELF/dtlto/imports.test
@@ -0,0 +1,53 @@
+# REQUIRES: x86
+
+# Check that DTLTO creates imports lists if requested.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: opt -thinlto-bc t1.ll -o t1.bc -O2
+RUN: opt -thinlto-bc t2.ll -o t2.bc -O2
+
+# Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o t1.o --relocation-model=pic
+RUN: llc t2.ll --filetype=obj -o t2.o --relocation-model=pic
+
+# Common command-line arguments. Note that mock.py does not do any compilation;
+# instead, it simply writes the contents of the object files supplied on the
+# command line into the output object files in job order.
+RUN: echo "t1.bc t2.bc \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=t1.o \
+RUN:   --thinlto-distributor-arg=t2.o" > l.rsp
+
+# Check that imports files are not created normally.
+RUN: ld.lld @l.rsp
+RUN: ls | FileCheck %s --check-prefix=NOIMPORTSFILES
+NOIMPORTSFILES-NOT: .imports
+
+# Check that imports files are created with --thinlto-emit-imports-files.
+RUN: ld.lld @l.rsp --thinlto-emit-imports-files
+RUN: ls | FileCheck %s --check-prefix=IMPORTSFILES
+IMPORTSFILES: t1.bc.imports
+IMPORTSFILES: t2.bc.imports
+
+#--- t1.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @t1() {
+entry:
+  ret void
+}
+
+#--- t2.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @t1(...)
+
+define void @t2() {
+entry:
+  call void (...) @t1()
+  ret void
+}
diff --git a/lld/test/ELF/dtlto/index.test b/lld/test/ELF/dtlto/index.test
@@ -0,0 +1,44 @@
+# REQUIRES: x86
+
+# Check that DTLTO creates individual summary index files if requested.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+# Generate ThinLTO bitcode files.
+RUN: opt -thinlto-bc t1.ll -o t1.bc -O2
+RUN: opt -thinlto-bc t1.ll -o t2.bc -O2
+
+# Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o t1.o --relocation-model=pic
+
+# Use a response file for the common command-line arguments.
+# Note that mock.py does not perform any compilation; instead, it copies the
+# contents of the specified object files into the output object files, in the
+# order the jobs are received.
+# The "--start-lib/--end-lib" options are used to exercise the special case
+# where unused lazy object inputs result in empty index files.
+RUN: echo "t1.bc --start-lib t2.bc --end-lib \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=t1.o \
+RUN:   --thinlto-distributor-arg=t2.o" > l.rsp
+
+# Check that index files are not created normally.
+RUN: ld.lld @l.rsp
+RUN: ls | FileCheck %s --check-prefix=NOINDEXFILES
+NOINDEXFILES-NOT: .thinlto.bc
+
+# Check that index files are created with --thinlto-emit-index-files.
+RUN: ld.lld @l.rsp --thinlto-emit-index-files
+RUN: ls | FileCheck %s --check-prefix=INDEXFILES
+INDEXFILES: t1.bc.thinlto.bc
+INDEXFILES: t2.bc.thinlto.bc
+
+#--- t1.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @t1() {
+entry:
+  ret void
+}
diff --git a/lld/test/ELF/dtlto/options.test b/lld/test/ELF/dtlto/options.test
@@ -0,0 +1,41 @@
+# REQUIRES: x86
+
+# Test that DTLTO options are passed correctly to the distributor and
+# remote compiler.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: opt -thinlto-bc foo.ll -o foo.o
+
+# Note: validate.py does not perform any compilation. Instead, it validates the
+# received JSON, pretty-prints the JSON and the supplied arguments, and then
+# exits with an error. This allows FileCheck directives to verify the
+# distributor inputs.
+RUN: not ld.lld foo.o \
+RUN:   -o my.elf \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \
+RUN:   --thinlto-distributor-arg=darg1=10 \
+RUN:   --thinlto-distributor-arg=darg2=20 \
+RUN:   --thinlto-remote-compiler=my_clang.exe \
+RUN:   --thinlto-remote-compiler-arg=carg1=20 \
+RUN:   --thinlto-remote-compiler-arg=carg2=30 2>&1 | FileCheck %s
+
+CHECK: distributor_args=['darg1=10', 'darg2=20']
+
+CHECK: "linker_output": "my.elf"
+
+CHECK: "my_clang.exe"
+CHECK: "carg1=20"
+CHECK: "carg2=30"
+
+CHECK: ld.lld: error: DTLTO backend compilation: cannot open native object file:
+
+#--- foo.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo() {
+entry:
+  ret void
+}
diff --git a/lld/test/ELF/dtlto/partitions.test b/lld/test/ELF/dtlto/partitions.test
diff --git a/lld/test/ELF/dtlto/save-temps.test b/lld/test/ELF/dtlto/save-temps.test
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+Tests for DTLTO (Integrated Distributed ThinLTO) functionality.`
	`2`	`+`
	`3`	+These are integration tests as DTLTO invokes `clang` for code-generation.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+if "clang" not in config.available_features:`
	`2`	`+ config.unsupported = True`