diff --git a/clang/docs/ThinLTO.rst b/clang/docs/ThinLTO.rst index c042547678919..687795ac655a7 100644 --- a/clang/docs/ThinLTO.rst +++ b/clang/docs/ThinLTO.rst @@ -240,6 +240,38 @@ The ``BOOTSTRAP_LLVM_ENABLE_LTO=Thin`` will enable ThinLTO for stage 2 and stage 3 in case the compiler used for stage 1 does not support the ThinLTO option. +Integrated Distributed ThinLTO (DTLTO) +-------------------------------------- + +Integrated Distributed ThinLTO (DTLTO) enables the distribution of backend +ThinLTO compilations via external distribution systems, such as Incredibuild, +during the traditional link step. + +The implementation is documented here: https://llvm.org/docs/DTLTO.html. + +DTLTO requires the LLD linker (``-fuse-ld=lld``). + +``-fthinlto-distributor=`` + - Specifies the ```` to the distributor process executable for DTLTO. + - If specified, ThinLTO backend compilations will be distributed by LLD. + +``-Xthinlto-distributor=`` + - Passes ```` to the distributor process (see ``-fthinlto-distributor=``). + - Can be specified multiple times to pass multiple options. + - Multiple options can also be specified by separating them with commas. + +Examples: + - ``clang -flto=thin -fthinlto-distributor=incredibuild.exe -Xthinlto-distributor=--verbose,--j10 -fuse-ld=lld`` + - ``clang -flto=thin -fthinlto-distributor=$(which python) -Xthinlto-distributor=incredibuild.py -fuse-ld=lld`` + +If ``-fthinlto-distributor=`` is specified, Clang supplies the path to a +compiler to be executed remotely to perform the ThinLTO backend +compilations. Currently, this is Clang itself. + +Note that currently, DTLTO is only supported in some LLD flavors. Support will +be added to other LLD flavours in the future. +See `DTLTO `_ for more information. + More Information ================ diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 0c8a219b19bf4..9c6f77af97be0 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -990,6 +990,13 @@ def Xlinker : Separate<["-"], "Xlinker">, Flags<[LinkerInput, RenderAsInput]>, Visibility<[ClangOption, CLOption, FlangOption]>, HelpText<"Pass to the linker">, MetaVarName<"">, Group; +def Xthinlto_distributor_EQ : CommaJoined<["-"], "Xthinlto-distributor=">, + Flags<[LinkOption]>, + Visibility<[ClangOption, CLOption]>, + HelpText<"Pass to the ThinLTO distributor process. Can be specified " + "multiple times or with comma-separated values.">, + MetaVarName<"">, + Group; def Xoffload_linker : JoinedAndSeparate<["-"], "Xoffload-linker">, Visibility<[ClangOption, FlangOption]>, HelpText<"Pass to the offload linkers or the ones identified by -">, @@ -4233,7 +4240,12 @@ def ffinite_loops: Flag<["-"], "ffinite-loops">, Group, def fno_finite_loops: Flag<["-"], "fno-finite-loops">, Group, HelpText<"Do not assume that any loop is finite.">, Visibility<[ClangOption, CC1Option]>; - +def fthinlto_distributor_EQ : Joined<["-"], "fthinlto-distributor=">, + Group, + HelpText<"Path to the ThinLTO distributor process. If specified, " + "ThinLTO backend compilations will be distributed by LLD">, + MetaVarName<"">, + Visibility<[ClangOption, CLOption]>; def ftrigraphs : Flag<["-"], "ftrigraphs">, Group, HelpText<"Process trigraph sequences">, Visibility<[ClangOption, CC1Option]>; def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group, diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp index f5e2655857432..3e9ca8f79d160 100644 --- a/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/clang/lib/Driver/ToolChains/Gnu.cpp @@ -455,6 +455,21 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA, addLTOOptions(ToolChain, Args, CmdArgs, Output, Inputs, D.getLTOMode() == LTOK_Thin); + // Forward the DTLTO options to the linker. We add these unconditionally, + // rather than in addLTOOptions() as it is the linker that decides whether to + // do LTO or not dependent upon whether there are any bitcode input files in + // the link. + if (Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) { + CmdArgs.push_back( + Args.MakeArgString("--thinlto-distributor=" + Twine(A->getValue()))); + CmdArgs.push_back( + Args.MakeArgString("--thinlto-remote-compiler=" + + Twine(ToolChain.getDriver().getClangProgramPath()))); + + for (auto A : Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ)) + CmdArgs.push_back(Args.MakeArgString("--thinlto-distributor-arg=" + A)); + } + if (Args.hasArg(options::OPT_Z_Xlinker__no_demangle)) CmdArgs.push_back("--no-demangle"); diff --git a/clang/test/Driver/DTLTO/dtlto.c b/clang/test/Driver/DTLTO/dtlto.c new file mode 100644 index 0000000000000..a23a10fdfb055 --- /dev/null +++ b/clang/test/Driver/DTLTO/dtlto.c @@ -0,0 +1,43 @@ +// REQUIRES: lld + +/// Check DTLTO options are forwarded to the linker. + +// RUN: echo "--target=x86_64-linux-gnu \ +// RUN: -Xthinlto-distributor=distarg1 \ +// RUN: -Xthinlto-distributor=distarg2,distarg3 \ +// RUN: -fuse-ld=lld" > %t.rsp + +/// Check that options are forwarded as expected with --thinlto-distributor=. +// RUN: %clang -### @%t.rsp -fthinlto-distributor=dist.exe %s 2>&1 | \ +// RUN: FileCheck %s --implicit-check-not=warning + +// CHECK: ld.lld +// CHECK-SAME: "--thinlto-distributor=dist.exe" +// CHECK-SAME: "--thinlto-remote-compiler={{.*}}clang +// CHECK-SAME: "--thinlto-distributor-arg=distarg1" +// CHECK-SAME: "--thinlto-distributor-arg=distarg2" +// CHECK-SAME: "--thinlto-distributor-arg=distarg3" + + +/// Check that options are not added without --thinlto-distributor= and +/// that there is an unused option warning issued for -Xthinlto-distributor= +/// options. We specify -flto here as these options should be unaffected by it. +// RUN: %clang -### @%t.rsp -flto=thin %s 2>&1 | \ +// RUN: FileCheck %s --check-prefixes=NONE,NOMORE --implicit-check-not=warning + +// NONE: warning: argument unused during compilation: '-Xthinlto-distributor=distarg1' +// NONE: warning: argument unused during compilation: '-Xthinlto-distributor=distarg2,distarg3' +// NONE: ld.lld +// NOMORE-NOT: distributor +// NOMORE-NOT: remote-compiler + + +/// Check the expected arguments are forwarded by default with only +/// --thinlto-distributor=. +// RUN: %clang --target=x86_64-linux-gnu -fthinlto-distributor=dist.exe \ +// RUN: -fuse-ld=lld -Werror -### %s 2>&1 | \ +// RUN: FileCheck %s --check-prefixes=DEFAULT,NOMORE + +// DEFAULT: ld.lld +// DEFAULT-SAME: "--thinlto-distributor=dist.exe" +// DEFAULT-SAME: "--thinlto-remote-compiler={{.*}}clang diff --git a/cross-project-tests/dtlto/ld-dtlto.c b/cross-project-tests/dtlto/ld-dtlto.c index 3ee962346bd4a..7dffe2e015bcb 100644 --- a/cross-project-tests/dtlto/ld-dtlto.c +++ b/cross-project-tests/dtlto/ld-dtlto.c @@ -5,13 +5,11 @@ // RUN: rm -rf %t && mkdir %t && cd %t -// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s -o dtlto.o - -// RUN: ld.lld dtlto.o \ -// RUN: --thinlto-distributor=%python \ -// RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ -// RUN: --thinlto-remote-compiler=%clang \ -// RUN: --thinlto-remote-compiler-arg=--save-temps +// RUN: %clang --target=x86_64-linux-gnu %s -flto=thin -fuse-ld=lld \ +// RUN: -fthinlto-distributor=%python \ +// RUN: -Xthinlto-distributor=%llvm_src_root/utils/dtlto/local.py \ +// RUN: -Wl,--thinlto-remote-compiler-arg=--save-temps \ +// RUN: -nostdlib -Werror /// Check that the required output files have been created. // RUN: ls | sort | FileCheck %s @@ -22,18 +20,15 @@ /// Linked ELF. // CHECK: {{^}}a.out{{$}} -/// Produced by the bitcode compilation. -// CHECK-NEXT: {{^}}dtlto.o{{$}} - /// --save-temps output for the backend compilation. -// CHECK-NEXT: {{^}}dtlto.s{{$}} -// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}} -// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP:[a-zA-Z0-9_]+]].s{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.0.preopt.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.1.promote.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.2.internalize.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.3.import.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.4.opt.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.5.precodegen.bc{{$}} +// CHECK-NEXT: {{^}}ld-dtlto-[[TMP]].s.resolution.txt{{$}} /// No files are expected after. // CHECK-NOT: {{.}}