diff --git a/clang/docs/ThinLTO.rst b/clang/docs/ThinLTO.rst index c042547678919..59366859d65cf 100644 --- a/clang/docs/ThinLTO.rst +++ b/clang/docs/ThinLTO.rst @@ -240,6 +240,39 @@ The ``BOOTSTRAP_LLVM_ENABLE_LTO=Thin`` will enable ThinLTO for stage 2 and stage 3 in case the compiler used for stage 1 does not support the ThinLTO option. +Distributed ThinLTO (DTLTO) +--------------------------- + +DTLTO allows for the distribution of backend ThinLTO compilations via external +distribution systems, e.g. Incredibuild. There is existing support for +distributing ThinLTO compilations by using separate thin-link, backend +compilation, and link steps coordinated by a build system which can handle the +dynamic dependencies specified by the index files, such as Bazel. However, this +often requires changes to the user's build process. With DTLTO distribution is +managed internally in LLD as part of the traditional link step and therefore +should be usable in any build process that can support in-process ThinLTO. + +DTLTO requires the LLD linker (``-fuse-ld=lld``). + +``-fthinlto-distributor=`` + - Specifies the ```` to the distributor process executable for DTLTO. + - If specified, ThinLTO backend compilations will be distributed by LLD. + +``-Xthinlto-distributor=`` + - Pass ```` to the distributor process (see ``-fthinlto-distributor=``). + - Can be specified multiple times to pass multiple options. + - Can specify multiple options by separating them with commas. + +Examples: + - ``clang -flto=thin -fthinlto-distributor=incredibuild.exe -Xthinlto-distributor=--verbose,--j10 -fuse-ld=lld`` + - ``clang -flto=thin -fthinlto-distributor=$(which python) -Xthinlto-distributor=incredibuild.py -fuse-ld=lld`` + +If ``-fthinlto-distributor=`` is specified Clang supplies the path to a compiler +to be exectuted remotely to perform the ThinLTO backend compilations. Currently +this is Clang itself. + +See `DTLTO `_ for more information. + More Information ================ diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 618815db28434..98e7952a751be 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -969,6 +969,10 @@ def Xlinker : Separate<["-"], "Xlinker">, Flags<[LinkerInput, RenderAsInput]>, Visibility<[ClangOption, CLOption, FlangOption]>, HelpText<"Pass to the linker">, MetaVarName<"">, Group; +def Xthinlto_distributor_EQ : CommaJoined<["-"], "Xthinlto-distributor=">, Flags<[LinkOption]>, + Visibility<[ClangOption, CLOption]>, + HelpText<"Pass to the ThinLTO distributor">, + MetaVarName<"">, Group; def Xoffload_linker : JoinedAndSeparate<["-"], "Xoffload-linker">, Visibility<[ClangOption, FlangOption]>, HelpText<"Pass to the offload linkers or the ones identified by -">, @@ -4087,7 +4091,9 @@ def ffinite_loops: Flag<["-"], "ffinite-loops">, Group, def fno_finite_loops: Flag<["-"], "fno-finite-loops">, Group, HelpText<"Do not assume that any loop is finite.">, Visibility<[ClangOption, CC1Option]>; - +def fthinlto_distributor_EQ : Joined<["-"], "fthinlto-distributor=">, Group, + HelpText<"Specifies the to the distributor process executable">, MetaVarName<"">, + Visibility<[ClangOption, CLOption]>; def ftrigraphs : Flag<["-"], "ftrigraphs">, Group, HelpText<"Process trigraph sequences">, Visibility<[ClangOption, CC1Option]>; def fno_trigraphs : Flag<["-"], "fno-trigraphs">, Group, diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp index f56eeda3cb5f6..71f94b3e8835e 100644 --- a/clang/lib/Driver/ToolChains/Gnu.cpp +++ b/clang/lib/Driver/ToolChains/Gnu.cpp @@ -535,6 +535,28 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA, D.getLTOMode() == LTOK_Thin); } + // Forward the DTLTO options to the linker. We add these unconditionally, + // rather than in addLTOOptions() as it is the linker that decides whether to + // do LTO or not dependent upon whether there are any bitcode input files in + // the link. + if (Arg *A = Args.getLastArg(options::OPT_fthinlto_distributor_EQ)) { + CmdArgs.push_back( + Args.MakeArgString("--thinlto-distributor=" + Twine(A->getValue()))); + CmdArgs.push_back( + Args.MakeArgString("-mllvm=-thinlto-remote-compiler=" + + Twine(ToolChain.getDriver().getClangProgramPath()))); + + for (auto A : Args.getAllArgValues(options::OPT_Xthinlto_distributor_EQ)) + CmdArgs.push_back( + Args.MakeArgString("-mllvm=-thinlto-distributor-arg=" + A)); + + for (const Arg *A : Args.filtered(options::OPT_mllvm)) { + CmdArgs.push_back(Args.MakeArgString( + Twine("-mllvm=-thinlto-remote-compiler-arg=-mllvm=") + + A->getValue(0))); + } + } + if (Args.hasArg(options::OPT_Z_Xlinker__no_demangle)) CmdArgs.push_back("--no-demangle"); diff --git a/clang/test/Driver/DTLTO/dtlto.c b/clang/test/Driver/DTLTO/dtlto.c new file mode 100644 index 0000000000000..b6613301baa57 --- /dev/null +++ b/clang/test/Driver/DTLTO/dtlto.c @@ -0,0 +1,50 @@ +/// Check DTLTO options are forwarded to the linker. + +// REQUIRES: lld + +// RUN: echo "--target=x86_64-linux-gnu \ +// RUN: -Xthinlto-distributor=distarg1 \ +// RUN: -Xthinlto-distributor=distarg2,distarg3 \ +// RUN: -mllvm llvm1 \ +// RUN: -mllvm=llvm2 \ +// RUN: -fuse-ld=lld" > %t.rsp + + +/// Check that options are forwarded as expected with --thinlto-distributor=. +// RUN: %clang -### @%t.rsp -fthinlto-distributor=dist.exe %s 2>&1 | \ +// RUN: FileCheck %s --implicit-check-not=warning + +// CHECK: ld.lld +// CHECK-SAME: "--thinlto-distributor=dist.exe" +// CHECK-SAME: "-mllvm=-thinlto-remote-compiler={{.*}}clang +// CHECK-SAME: "-mllvm=-thinlto-distributor-arg=distarg1" +// CHECK-SAME: "-mllvm=-thinlto-distributor-arg=distarg2" +// CHECK-SAME: "-mllvm=-thinlto-distributor-arg=distarg3" +// CHECK-SAME: "-mllvm=-thinlto-remote-compiler-arg=-mllvm=llvm1" +// CHECK-SAME: "-mllvm=-thinlto-remote-compiler-arg=-mllvm=llvm2" + + +/// Check that options are not added without --thinlto-distributor= and +/// that there is an unused option warning issued for -Xthinlto-distributor= options. We +/// specify -flto here as these options should be unaffected by it. +// RUN: %clang -### @%t.rsp -flto=thin %s 2>&1 | \ +// RUN: FileCheck %s --check-prefixes=NONE,NOMORE --implicit-check-not=warning + +// NONE: warning: argument unused during compilation: '-Xthinlto-distributor=distarg1' +// NONE: warning: argument unused during compilation: '-Xthinlto-distributor=distarg2,distarg3' +// NONE: ld.lld +// NOMORE-NOT: --thinlto-distributor= +// NOMORE-NOT: -thinlto-remote-compiler= +// NOMORE-NOT: -mllvm +// NOMORE-NOT: -thinlto-distributor-arg= + + +/// Check the expected arguments are forwarded by default with only +/// --thinlto-distributor=. +// RUN: %clang --target=x86_64-linux-gnu -fthinlto-distributor=dist.exe \ +// RUN: -fuse-ld=lld -Werror -### %s 2>&1 | \ +// RUN: FileCheck %s --check-prefixes=DEFAULT,NOMORE + +// DEFAULT: ld.lld +// DEFAULT-SAME: "--thinlto-distributor=dist.exe" +// DEFAULT-SAME: "-mllvm=-thinlto-remote-compiler={{.*}}clang diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt index 7f2fee48fda77..25f03ce88fd78 100644 --- a/cross-project-tests/CMakeLists.txt +++ b/cross-project-tests/CMakeLists.txt @@ -19,11 +19,15 @@ set(CROSS_PROJECT_TEST_DEPS FileCheck check-gdb-llvm-support count - llvm-dwarfdump + llvm-ar llvm-config + llvm-dwarfdump + llvm-lto2 llvm-objdump - split-file + llvm-profdata not + opt + split-file ) if ("clang" IN_LIST LLVM_ENABLE_PROJECTS) @@ -94,6 +98,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests" DEPENDS clang ) +# DTLTO tests. +add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests" + ${CMAKE_CURRENT_BINARY_DIR}/dtlto + EXCLUDE_FROM_CHECK_ALL + DEPENDS ${CROSS_PROJECT_TEST_DEPS} + ) + # Add check-cross-project-* targets. add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${CROSS_PROJECT_TEST_DEPS} diff --git a/cross-project-tests/dtlto/README.md b/cross-project-tests/dtlto/README.md new file mode 100644 index 0000000000000..cfd9d3496ca42 --- /dev/null +++ b/cross-project-tests/dtlto/README.md @@ -0,0 +1,3 @@ +Tests for DTLTO (integrated distributed ThinLTO) functionality. + +These are integration tests as DTLTO invokes `clang` for code-generation. \ No newline at end of file diff --git a/cross-project-tests/dtlto/archive-thin.test b/cross-project-tests/dtlto/archive-thin.test new file mode 100644 index 0000000000000..4f8ef423aca19 --- /dev/null +++ b/cross-project-tests/dtlto/archive-thin.test @@ -0,0 +1,72 @@ +## Simple test that a DTLTO link succeeds and outputs the expected set of files +## correctly when thin archives are present. + +# RUN: rm -rf %t.dir && split-file %s %t.dir && cd %t.dir +# RUN: %clang --target=x86_64-linux-gnu -c foo.c -o foo.o +# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin bar.c -o bar.o +# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin dog.c -o dog.o +# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin cat.c -o cat.o +# RUN: %clang --target=x86_64-linux-gnu -c -flto=thin _start.c -o _start.o + +# RUN: llvm-ar rcs foo.a foo.o --thin +## Create this bitcode thin archive in a sub-directory to test the expansion of +## the path to a bitcode file which is referenced using "..", e.g. in this case +## "../bar.o". The ".." should be collapsed in any expansion to avoid +## referencing an unknown directory on the remote side. +# RUN: mkdir lib +# RUN: llvm-ar rcs lib/bar.a bar.o --thin +## Create this bitcode thin archive with an absolute path entry containing "..". +# RUN: llvm-ar rcs dog.a %t.dir/lib/../dog.o --thin +# RUN: llvm-ar rcs cat.a cat.o --thin +# RUN: llvm-ar rcs _start.a _start.o --thin + +# RUN: mkdir %t.dir/out && cd %t.dir/out + +# RUN: %clang --target=x86_64-linux-gnu \ +# RUN: %t.dir/foo.a %t.dir/lib/bar.a ../_start.a %t.dir/cat.a -Wl,--whole-archive,../dog.a \ +# RUN: -flto=thin \ +# RUN: -fthinlto-distributor=%python \ +# RUN: -Xthinlto-distributor=%llvm_src_root/utils/dtlto/local.py \ +# RUN: --save-temps \ +# RUN: -fuse-ld=lld \ +# RUN: -nostdlib \ +# RUN: -nostartfiles \ +# RUN: -Wl,--save-temps \ +# RUN: -Wl,-mllvm,--thinlto-remote-compiler-arg=-save-temps=cwd \ +# RUN: -Werror + +## Check that the required output files have been created. +# RUN: ls | FileCheck %s --check-prefix=OUTPUTS \ +# RUN: --implicit-check-not=cat --implicit-check-not=foo + +## The DTLTO backend emits the JSON jobs description and summary shards. +# OUTPUTS-DAG: a.{{[0-9]+}}.dist-file.json +# OUTPUTS-DAG: bar.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}} +# OUTPUTS-DAG: dog.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}} +# OUTPUTS-DAG: _start.{{[0-9]+}}.{{[0-9]+}}.native.o.thinlto.bc{{$}} +## Native output object files. +# OUTPUTS-DAG: bar.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}} +# OUTPUTS-DAG: dog.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}} +# OUTPUTS-DAG: _start.{{[0-9]+}}.{{[0-9]+}}.native.o{{$}} + +## Check that bar.o and dog.o are not referenced using "..". +# RUN: not grep '\.\.\(/\|\\\\\)\(bar\|dog\)\.o' a.*.dist-file.json + +#--- foo.c +__attribute__((retain)) void foo() {} + +#--- bar.c +extern void foo(); +__attribute__((retain)) void bar() { foo(); } + +#--- dog.c +__attribute__((retain)) void dog() {} + +#--- cat.c +__attribute__((retain)) void cat() {} + +#--- _start.c +extern void bar(); +__attribute__((retain)) void _start() { + bar(); +} diff --git a/cross-project-tests/dtlto/dtlto-translate-options.ll b/cross-project-tests/dtlto/dtlto-translate-options.ll new file mode 100644 index 0000000000000..7f5888cf9e6c7 --- /dev/null +++ b/cross-project-tests/dtlto/dtlto-translate-options.ll @@ -0,0 +1,142 @@ +;; Check that the expected Clang arguments are generated by DTLTO for the +;; backend compilations and are accepted by Clang. + +; RUN: rm -rf %t && split-file %s %t && cd %t + +;; Generate bitcode files with a summary index. +; RUN: opt -thinlto-bc x86_64-unknown-linux-gnu.ll -o x86_64-unknown-linux-gnu.bc +; RUN: opt -thinlto-bc x86_64-pc-windows-msvc.ll -o x86_64-pc-windows-msvc.bc + + +;; Check that invalid arguments cause a Clang error. This property is relied on +;; by the actual testcases later in this test. +; RUN: not %clang -x ir x86_64-unknown-linux-gnu.ll \ +; RUN: -invalid-incorrect-not-an-option 2>&1 | FileCheck %s --check-prefix=SANITY1 +; SANITY1: unknown argument: '-invalid-incorrect-not-an-option' + + +;; Define a substitution used to simplify the testcases. +; DEFINE: %{distributor} = dummy +; DEFINE: %{extra_flags} = dummy +; DEFINE: %{triple} = dummy +; DEFINE: %{command} = llvm-lto2 run \ +; DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/%{distributor} \ +; DEFINE: -thinlto-remote-compiler-arg=-Wunused-command-line-argument \ +; DEFINE: @%{triple}.rsp %{extra_flags} + + +;; Write common arguments to response files. + +; RUN: echo "x86_64-unknown-linux-gnu.bc -o x86_64-unknown-linux-gnu.o \ +; RUN: -dtlto \ +; RUN: -dtlto-distributor=%python \ +; RUN: -thinlto-remote-compiler=%clang \ +; RUN: -thinlto-remote-compiler-arg=-Werror \ +; RUN: -r=x86_64-unknown-linux-gnu.bc,globalfunc1,plx" > x86_64-unknown-linux-gnu.rsp + +; RUN: echo "x86_64-pc-windows-msvc.bc -o x86_64-pc-windows-msvc.o \ +; RUN: -dtlto \ +; RUN: -dtlto-distributor=%python \ +; RUN: -thinlto-remote-compiler=%clang \ +; RUN: -thinlto-remote-compiler-arg=-Werror \ +; RUN: -thinlto-remote-compiler-arg=-Wno-override-module \ +; RUN: -r=x86_64-pc-windows-msvc.bc,globalfunc2,plx" > x86_64-pc-windows-msvc.rsp + + +;; Check that boolean configuration states are translated as expected and Clang +;; accepts them. + +; RUN: echo " \ +; RUN: --addrsig=1 \ +; RUN: -function-sections=1 \ +; RUN: -data-sections=1" > on.rsp + +; RUN: echo " \ +; RUN: --addrsig=0 \ +; RUN: -function-sections=0 \ +; RUN: -data-sections=0" > off.rsp + +;; Perform DTLTO with configuration state set. +; REDEFINE: %{extra_flags} = @on.rsp +; REDEFINE: %{distributor} = local.py +; REDEFINE: %{triple} = x86_64-unknown-linux-gnu +; RUN: %{command} +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=ON \ +; RUN: --implicit-check-not=-no-pgo-warn-mismatch +; ON-DAG: "-faddrsig" +; ON-DAG: "-ffunction-sections" +; ON-DAG: "-fdata-sections" + +;; Perform DTLTO with configuration state unset. +; REDEFINE: %{extra_flags} = @off.rsp +; REDEFINE: %{distributor} = local.py +; RUN: %{command} +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=OFF +; OFF-NOT: --implicit-check-not=--faddrsig +; OFF-NOT: --implicit-check-not=--ffunction-sections +; OFF-NOT: --implicit-check-not=--fdata-sections +; OFF-NOT: --implicit-check-not=-no-pgo-warn-mismatch + + +;; Check optimization level. + +; RUN: llvm-lto2 run \ +; RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ +; RUN: @x86_64-unknown-linux-gnu.rsp \ +; RUN: -O3 + +; RUN: not llvm-lto2 run \ +; RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \ +; RUN: @x86_64-unknown-linux-gnu.rsp \ +; RUN: -O3 2>&1 | FileCheck %s --check-prefix=OPTLEVEL +; OPTLEVEL-DAG: "-O3" + + +;; Check relocation model. + +; REDEFINE: %{extra_flags} = -relocation-model=pic +; REDEFINE: %{distributor} = local.py +; RUN: %{command} +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=PIC +; PIC: -fpic + + +; REDEFINE: %{extra_flags} = -relocation-model=pic +; REDEFINE: %{distributor} = local.py +; REDEFINE: %{triple} = x86_64-pc-windows-msvc +; RUN: %{command} +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=NOPIC +; REDEFINE: %{triple} = x86_64-unknown-linux-gnu +; NOPIC-NOT: -fpic + +;; Check specifying a sample profile. +; REDEFINE: %{extra_flags} = --lto-sample-profile-file="missing.profdata" +; REDEFINE: %{distributor} = local.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE_ERR +; SAMPLE_PROFILE_ERR: no such file or directory: 'missing.profdata' +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE +; SAMPLE_PROFILE-DAG: "-fprofile-sample-use=missing.profdata" + + +;--- x86_64-unknown-linux-gnu.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @globalfunc1() { +entry: + ret void +} + +;--- x86_64-pc-windows-msvc.ll +target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc" + +define void @globalfunc2() { +entry: + ret void +} diff --git a/cross-project-tests/dtlto/dtlto.c b/cross-project-tests/dtlto/dtlto.c new file mode 100644 index 0000000000000..3983d1cc51ae3 --- /dev/null +++ b/cross-project-tests/dtlto/dtlto.c @@ -0,0 +1,45 @@ +/// Simple test that DTLTO works with a single input file and generates the +/// expected set of files with --save-temps applied to the linker. +/// +/// Note that we also supply --save-temps to the compiler for predictable +/// bitcode file names. + +// RUN: rm -rf %t && mkdir %t && cd %t + +// RUN: %clang --target=x86_64-linux-gnu %s -shared -flto=thin \ +// RUN: -fthinlto-distributor=%python \ +// RUN: -Xthinlto-distributor=%llvm_src_root/utils/dtlto/local.py \ +// RUN: --save-temps \ +// RUN: -fuse-ld=lld \ +// RUN: -nostdlib \ +// RUN: -Wl,--save-temps \ +// RUN: -Werror + +/// Check that the required output files have been created. +// RUN: ls | count 13 +// RUN: ls | FileCheck %s + +/// Produced by the bitcode compilation. +// CHECK-DAG: {{^}}dtlto.bc{{$}} +// CHECK-DAG: {{^}}dtlto.i{{$}} +// CHECK-DAG: {{^}}dtlto.o{{$}} + +/// A jobs description JSON and a summary shard is emitted for DTLTO. +// CHECK-DAG: {{^}}a.[[#]].dist-file.json{{$}} +// CHECK-DAG: {{^}}dtlto.[[#]].[[#]].native.o.thinlto.bc{{$}} + +/// The backend compilation produces a native object output file for dtlto.o. +// CHECK-DAG: dtlto.[[#]].[[#]].native.o{{$}} + +/// Linked ELF. +// CHECK-DAG: {{^}}a.out{{$}} + +/// --save-temps incremental files for a.out. +// CHECK-DAG: {{^}}a.out.lto.dtlto.o{{$}} +// CHECK-DAG: {{^}}a.out.0.0.preopt.bc{{$}} +// CHECK-DAG: {{^}}a.out.0.2.internalize.bc{{$}} +// CHECK-DAG: {{^}}a.out.index.bc{{$}} +// CHECK-DAG: {{^}}a.out.index.dot{{$}} +// CHECK-DAG: {{^}}a.out.resolution.txt{{$}} + +int _start() { return 0; } diff --git a/cross-project-tests/dtlto/lit.local.cfg b/cross-project-tests/dtlto/lit.local.cfg new file mode 100644 index 0000000000000..222a1c98a9eba --- /dev/null +++ b/cross-project-tests/dtlto/lit.local.cfg @@ -0,0 +1,2 @@ +if any(feature not in config.available_features for feature in ["clang", "llvm-ar", "llvm-lto2", "opt"]): + config.unsupported = True diff --git a/cross-project-tests/dtlto/mllvm.c b/cross-project-tests/dtlto/mllvm.c new file mode 100644 index 0000000000000..881aafbd56bee --- /dev/null +++ b/cross-project-tests/dtlto/mllvm.c @@ -0,0 +1,33 @@ +// clang-format off +/// Test that -mllvm options are forwarded to the remote compiler for DTLTO. + +// RUN: rm -rf %t && mkdir %t && cd %t + +// RUN: %clang --target=x86_64-linux-gnu %s -shared -flto=thin \ +// RUN: -fthinlto-distributor=%python \ +// RUN: -Xthinlto-distributor=%llvm_src_root/utils/dtlto/local.py \ +// RUN: -fuse-ld=lld \ +// RUN: -nostdlib \ +// RUN: -Werror \ +/// Specify -v for both the inital and remote clang invocations. +// RUN: -v \ +// RUN: -Wl,-mllvm=-thinlto-remote-compiler-arg=-v \ +/// -thinlto-remote-compiler-arg is a safe -mllvm option to use for this test as +/// it is part of the DTLTO implemenation. +// RUN: -mllvm -thinlto-remote-compiler-arg=llvm1 \ +// RUN: -mllvm=-thinlto-remote-compiler-arg=llvm2 \ +// RUN: 2>&1 | FileCheck %s + +// -mllvm arguments are forwarded to `clang -cc1`. +// CHECK: -mllvm -thinlto-remote-compiler-arg=llvm1 +// CHECK-SAME: -mllvm -thinlto-remote-compiler-arg=llvm2 + +// -mllvm arguments are forwarded to the remote compiler via lld. +// CHECK: -mllvm=-thinlto-remote-compiler-arg=-mllvm=-thinlto-remote-compiler-arg=llvm1 +// CHECK-SAME: -mllvm=-thinlto-remote-compiler-arg=-mllvm=-thinlto-remote-compiler-arg=llvm2 + +// -mllvm arguments are forwarded to `clang -cc1` in the remote execution. +// CHECK: -mllvm -thinlto-remote-compiler-arg=llvm1 +// CHECK-SAME: -mllvm -thinlto-remote-compiler-arg=llvm2 + +int _start() { return 0; } diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py index 66fdd63632885..b77d26cb8ce28 100644 --- a/cross-project-tests/lit.cfg.py +++ b/cross-project-tests/lit.cfg.py @@ -19,7 +19,7 @@ config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell) # suffixes: A list of file extensions to treat as test files. -config.suffixes = [".c", ".cl", ".cpp", ".m"] +config.suffixes = [".c", ".cl", ".cpp", ".m", ".ll", ".test"] # excludes: A list of directories to exclude from the testsuite. The 'Inputs' # subdirectories contain auxiliary inputs for various tests in their parent @@ -96,6 +96,9 @@ def get_required_attr(config, attr_name): if lldb_path is not None: config.available_features.add("lldb") +for tool in ["llvm-ar", "llvm-lto2", "opt"]: + if llvm_config.use_llvm_tool(tool): + config.available_features.add(tool) def configure_dexter_substitutions(): """Configure substitutions for host platform and return list of dependencies""" diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h index 0c7c4e91402f1..19272569f3ce9 100644 --- a/lld/COFF/Config.h +++ b/lld/COFF/Config.h @@ -192,6 +192,9 @@ struct Configuration { // Used for /lldltocachepolicy=policy llvm::CachePruningPolicy ltoCachePolicy; + // Used for --thinlto-distributor= + StringRef dtltoDistributor; + // Used for /opt:[no]ltodebugpassmanager bool ltoDebugPassManager = false; diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp index 6433ce6643f9c..fe766e89a2364 100644 --- a/lld/COFF/Driver.cpp +++ b/lld/COFF/Driver.cpp @@ -1518,6 +1518,14 @@ void LinkerDriver::linkerMain(ArrayRef argsArr) { v.push_back(arg->getValue()); config->mllvmOpts.emplace_back(arg->getValue()); } + + if (!ctx.config.dtltoDistributor.empty()) + for (auto o : {"-thinlto-remote-compiler-arg=-fdiagnostics-format", + "-thinlto-remote-compiler-arg=msvc"}) { + v.push_back(o); + config->mllvmOpts.emplace_back(o); + } + { llvm::TimeTraceScope timeScope2("Parse cl::opt"); cl::ResetAllOptionOccurrences(); @@ -2081,6 +2089,9 @@ void LinkerDriver::linkerMain(ArrayRef argsArr) { Fatal(ctx) << "/manifestinput: requires /manifest:embed"; } + // Handle DTLTO options. + config->dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq); + // Handle /dwodir config->dwoDir = args.getLastArgValue(OPT_dwodir); diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index a8cecb39ac614..3f38e395eb884 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -16,6 +16,7 @@ #include "lld/Common/Filesystem.h" #include "lld/Common/Strings.h" #include "lld/Common/TargetOptionsCommandFlags.h" +#include "lld/Common/Version.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" @@ -116,7 +117,14 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { // Initialize ltoObj. lto::ThinBackend backend; - if (ctx.config.thinLTOIndexOnly) { + if (!ctx.config.dtltoDistributor.empty()) { + backend = lto::createOutOfProcessThinBackend( + llvm::heavyweight_hardware_concurrency(ctx.config.thinLTOJobs), + /*OnWrite=*/nullptr, + /*ShouldEmitIndexFiles=*/false, + /*ShouldEmitImportFiles=*/false, ctx.config.outputFile, + ctx.config.dtltoDistributor, !ctx.config.saveTempsArgs.empty()); + } else if (ctx.config.thinLTOIndexOnly) { auto OnIndexWrite = [&](StringRef S) { thinIndices.erase(S); }; backend = lto::createWriteIndexesThinBackend( llvm::hardware_concurrency(ctx.config.thinLTOJobs), @@ -182,13 +190,15 @@ std::vector BitcodeCompiler::compile() { // native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. FileCache cache; + auto AddBuffer = [&](size_t task, const Twine &moduleName, + std::unique_ptr mb) { + files[task] = std::move(mb); + file_names[task] = moduleName.str(); + }; + if (!ctx.config.ltoCache.empty()) - cache = check(localCache("ThinLTO", "Thin", ctx.config.ltoCache, - [&](size_t task, const Twine &moduleName, - std::unique_ptr mb) { - files[task] = std::move(mb); - file_names[task] = moduleName.str(); - })); + cache = + check(localCache("ThinLTO", "Thin", ctx.config.ltoCache, AddBuffer)); checkError(ltoObj->run( [&](size_t task, const Twine &moduleName) { @@ -196,7 +206,7 @@ std::vector BitcodeCompiler::compile() { return std::make_unique( std::make_unique(buf[task].second)); }, - cache)); + cache, AddBuffer)); // Emit empty index files for non-indexed files for (StringRef s : thinIndices) { diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td index b6fd3d0daaef9..8b75a91b5e832 100644 --- a/lld/COFF/Options.td +++ b/lld/COFF/Options.td @@ -270,6 +270,8 @@ def thinlto_object_suffix_replace : P< def thinlto_prefix_replace: P< "thinlto-prefix-replace", "'old;new' replace old prefix with new prefix in ThinLTO outputs">; +def thinlto_distributor_eq: Joined<["--"], "thinlto-distributor=">, + HelpText<"Distributor to use for ThinLTO backend compilations">; def lto_obj_path : P< "lto-obj-path", "output native object for merged LTO unit to this path">; diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h index f132b11b20c63..b95cb81efae3b 100644 --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -243,6 +243,7 @@ struct Config { llvm::SmallVector searchPaths; llvm::SmallVector symbolOrderingFile; llvm::SmallVector thinLTOModulesToCompile; + llvm::StringRef dtltoDistributor; llvm::SmallVector undefined; llvm::SmallVector dynamicList; llvm::SmallVector buildIdVector; diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index 7d14180a49926..f688c7fda165e 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -1341,6 +1341,7 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) { args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true); ctx.arg.disableVerify = args.hasArg(OPT_disable_verify); ctx.arg.discard = getDiscard(args); + ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq); ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq); ctx.arg.dynamicLinker = getDynamicLinker(ctx, args); ctx.arg.ehFrameHdr = diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index d43de8ce6dfef..aaa8346f7fc99 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -21,6 +21,7 @@ #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/STLExtras.h" #include "llvm/LTO/LTO.h" +#include "llvm/Object/Archive.h" #include "llvm/Object/IRObjectFile.h" #include "llvm/Support/ARMAttributeParser.h" #include "llvm/Support/ARMBuildAttributes.h" @@ -1702,6 +1703,38 @@ static uint8_t getOsAbi(const Triple &t) { } } +// For DTLTO, bitcode member names must be a valid path to a bitcode file on +// disk. For thin archives, adjust `memberPath` to the full file path of the +// archive member. Returns true if an adjustment was made; false otherwise. +// Non-thin archives are not yet supported. +static bool dtltoAdjustMemberPathIfThinArchive(Ctx &ctx, StringRef archivePath, + std::string &memberPath) { + assert(!archivePath.empty()); + assert(!ctx.arg.dtltoDistributor.empty()); + + // Check if the archive file is a thin archive by reading its header. + auto memBufferOrError = + MemoryBuffer::getFileSlice(archivePath, sizeof(ThinArchiveMagic) - 1, 0); + if (std::error_code ec = memBufferOrError.getError()) { + ErrAlways(ctx) << "cannot open " << archivePath << ": " << ec.message(); + return false; + } + MemoryBufferRef memBufRef = *memBufferOrError.get(); + if (!memBufRef.getBuffer().starts_with(ThinArchiveMagic)) + return false; + + SmallString<64> archiveMemberPath; + if (path::is_relative(memberPath)) { + archiveMemberPath = path::parent_path(archivePath); + path::append(archiveMemberPath, memberPath); + } else + archiveMemberPath = memberPath; + + path::remove_dots(archiveMemberPath, /*remove_dot_dot=*/true); + memberPath = archiveMemberPath.str(); + return true; +} + BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, uint64_t offsetInArchive, bool lazy) : InputFile(ctx, BitcodeKind, mb) { @@ -1711,7 +1744,6 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, std::string path = mb.getBufferIdentifier().str(); if (ctx.arg.thinLTOIndexOnly) path = replaceThinLTOSuffix(ctx, mb.getBufferIdentifier()); - // ThinLTO assumes that all MemoryBufferRefs given to it have a unique // name. If two archives define two members with the same name, this // causes a collision which result in only one of the objects being taken @@ -1719,10 +1751,13 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, // symbols later in the link stage). So we append file offset to make // filename unique. StringSaver &ss = ctx.saver; - StringRef name = archiveName.empty() - ? ss.save(path) - : ss.save(archiveName + "(" + path::filename(path) + - " at " + utostr(offsetInArchive) + ")"); + StringRef name = + (archiveName.empty() || + (!ctx.arg.dtltoDistributor.empty() && + dtltoAdjustMemberPathIfThinArchive(ctx, archiveName, path))) + ? ss.save(path) + : ss.save(archiveName + "(" + path::filename(path) + " at " + + utostr(offsetInArchive) + ")"); MemoryBufferRef mbref(mb.getBuffer(), name); obj = CHECK2(lto::InputFile::create(mbref), this); diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index 195526bf390d2..8391cfde3848c 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -17,6 +17,7 @@ #include "lld/Common/Filesystem.h" #include "lld/Common/Strings.h" #include "lld/Common/TargetOptionsCommandFlags.h" +#include "lld/Common/Version.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" @@ -186,6 +187,12 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) { std::string(ctx.arg.thinLTOPrefixReplaceNew), std::string(ctx.arg.thinLTOPrefixReplaceNativeObject), ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite); + } else if (!ctx.arg.dtltoDistributor.empty() && !ctx.bitcodeFiles.empty()) { + backend = lto::createOutOfProcessThinBackend( + llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs), + onIndexWrite, ctx.arg.thinLTOEmitIndexFiles, + ctx.arg.thinLTOEmitImportsFiles, ctx.arg.outputFile, + ctx.arg.dtltoDistributor, !ctx.arg.saveTempsArgs.empty()); } else { backend = lto::createInProcessThinBackend( llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs), @@ -319,13 +326,14 @@ SmallVector, 0> BitcodeCompiler::compile() { // to cache native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. FileCache cache; + AddBufferFn AddBuffer = [&](size_t task, const Twine &moduleName, + std::unique_ptr mb) { + files[task] = std::move(mb); + filenames[task] = moduleName.str(); + }; if (!ctx.arg.thinLTOCacheDir.empty()) - cache = check(localCache("ThinLTO", "Thin", ctx.arg.thinLTOCacheDir, - [&](size_t task, const Twine &moduleName, - std::unique_ptr mb) { - files[task] = std::move(mb); - filenames[task] = moduleName.str(); - })); + cache = check( + localCache("ThinLTO", "Thin", ctx.arg.thinLTOCacheDir, AddBuffer)); if (!ctx.bitcodeFiles.empty()) checkError(ctx.e, ltoObj->run( @@ -335,7 +343,7 @@ SmallVector, 0> BitcodeCompiler::compile() { std::make_unique( buf[task].second)); }, - cache)); + cache, AddBuffer)); // Emit empty index files for non-indexed files but not in single-module mode. if (ctx.arg.thinLTOModulesToCompile.empty()) { diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td index b3b12a0646875..20aa08747413b 100644 --- a/lld/ELF/Options.td +++ b/lld/ELF/Options.td @@ -700,7 +700,8 @@ def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">; def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">; def thinlto_single_module_eq: JJ<"thinlto-single-module=">, HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">; - +def thinlto_distributor_eq: JJ<"thinlto-distributor=">, + HelpText<"Distributor to use for ThinLTO backend compilations">; defm fat_lto_objects: BB<"fat-lto-objects", "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.", "Ignore the .llvm.lto section in relocatable object files (default).">; diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp index 2eeca44ecbb3c..feffa90127099 100644 --- a/lld/MachO/LTO.cpp +++ b/lld/MachO/LTO.cpp @@ -198,12 +198,13 @@ std::vector BitcodeCompiler::compile() { // to cache native object files for ThinLTO incremental builds. If a path was // specified, configure LTO to use it as the cache directory. FileCache cache; + AddBufferFn AddBuffer = [&](size_t task, const Twine &moduleName, + std::unique_ptr mb) { + files[task] = std::move(mb); + }; if (!config->thinLTOCacheDir.empty()) - cache = check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir, - [&](size_t task, const Twine &moduleName, - std::unique_ptr mb) { - files[task] = std::move(mb); - })); + cache = check( + localCache("ThinLTO", "Thin", config->thinLTOCacheDir, AddBuffer)); if (hasFiles) checkError(ltoObj->run( @@ -211,7 +212,7 @@ std::vector BitcodeCompiler::compile() { return std::make_unique( std::make_unique(buf[task])); }, - cache)); + cache, AddBuffer)); // Emit empty index files for non-indexed files for (StringRef s : thinIndices) { diff --git a/lld/docs/DTLTO.rst b/lld/docs/DTLTO.rst new file mode 100644 index 0000000000000..446f9cd522009 --- /dev/null +++ b/lld/docs/DTLTO.rst @@ -0,0 +1,59 @@ +Distributed ThinLTO (DTLTO) +=========================== + +DTLTO allows for the distribution of backend ThinLTO compilations via external +distribution systems, e.g. Incredibuild. There is existing support for +distributing ThinLTO compilations by using separate thin-link, backend +compilation, and link steps coordinated by a build system that can handle the +dynamic dependencies specified by the index files, such as Bazel. However, this +often requires changes to the user's build process. DTLTO distribution is +managed internally in LLD as part of the traditional link step and, therefore, +should be usable via any build process that can support in-process ThinLTO. + +ELF LLD +------- + +The command line interface for DTLTO is: + +- ``--thinlto-distributor=`` + Specifies the file to execute as a distributor process. + If specified, ThinLTO backend compilations will be distributed. + +- ``-mllvm -thinlto-remote-compiler=`` + Specifies the path to the compiler that the distributor process will use for + backend compilations. + + The compiler invoked must match the version of LLD. + + Currently `Clang` is used on remote machines to perform optimization. The + design permits this to be swapped out later without affecting distributors. + This may occur in the future, at which point a different set of constraints + will apply. + +- ``-mllvm -thinlto-distributor-arg=`` + Specifies `` on the command line when invoking the distributor. + +- ``-mllvm -thinlto-remote-compiler-arg=`` + Specifies `` on the command line to the remote compiler. These arguments + are appended to the end of the command line for the remote compiler. + +Remote compiler options that imply an additional input or output file dependency +are unsupported and may result in miscompilation depending on the properties of +the distribution system (as such additional input/output files may not be pushed +to or fetched from distribution system nodes correctly). If such options are +required, then the distributor can be modified to accept switches that specify +additional input/output dependencies, and ``-mllvm -thinlto-distributor-arg=`` +can be used to pass such options through to the distributor. + +Some LLD LTO options (e.g., ``--lto-sample-profile=``) are supported. +Currently, other options are silently accepted but do not have the desired +effect. Support for such options will be expanded in the future. + +COFF LLD +-------- + +The command line interface for COFF LLD is generally the same as for ELF LLD. +Note however, that ``mllvm`` options are passed using ``/mllvm: