[DTLTO] Normalize Windows 8.3 short paths to long form

cross-project-tests/CMakeLists.txt

@@ -19,11 +19,12 @@ set(CROSS_PROJECT_TEST_DEPS
   FileCheck
   check-gdb-llvm-support
   count
-  llvm-dwarfdump
+  llvm-ar
   llvm-config
+  llvm-dwarfdump
   llvm-objdump
-  split-file
   not
+  split-file
   )
 
 if ("clang" IN_LIST LLVM_ENABLE_PROJECTS)
@@ -94,6 +95,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
   DEPENDS clang
   )
 
+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${CROSS_PROJECT_TEST_DEPS}

cross-project-tests/dtlto/README.md

@@ -0,0 +1,3 @@
+Tests for DTLTO (Integrated Distributed ThinLTO) functionality.
+
+These are integration tests as DTLTO invokes `clang` for code-generation.

cross-project-tests/dtlto/archive-thin.test

@@ -0,0 +1,65 @@
+# REQUIRES: x86-registered-target,ld.lld,llvm-ar
+
+# Test that a DTLTO link succeeds and outputs the expected set of files
+# correctly when thin archives are present.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+RUN: %clang --target=x86_64-linux-gnu -flto=thin -c \
+RUN:   foo.c bar.c dog.c cat.c _start.c
+
+RUN: llvm-ar rcs foo.a foo.o --thin
+# Create this bitcode thin archive in a subdirectory to test the expansion of
+# the path to a bitcode file that is referenced using "..", e.g., in this case
+# "../bar.o".
+RUN: mkdir lib
+RUN: llvm-ar rcs lib/bar.a bar.o --thin
+# Create this bitcode thin archive with an absolute path entry containing "..".
+RUN: llvm-ar rcs dog.a %t/lib/../dog.o --thin
+RUN: llvm-ar rcs cat.a cat.o --thin
+RUN: llvm-ar rcs _start.a _start.o --thin
+
+RUN: mkdir %t/out && cd %t/out
+
+RUN: ld.lld %t/foo.a %t/lib/bar.a ../_start.a %t/cat.a \
+RUN:   --whole-archive ../dog.a \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+RUN:   --thinlto-remote-compiler=%clang \
+RUN:   --save-temps
+
+# Check that the required output files have been created.
+RUN: ls | FileCheck %s --check-prefix=OUTPUTS \
+RUN:     --implicit-check-not=cat --implicit-check-not=foo
+
+# JSON jobs description.
+OUTPUTS-DAG: a.[[PID:[a-zA-Z0-9_]+]].dist-file.json
+
+# Individual summary index files.
+OUTPUTS-DAG:    dog.1.[[PID]].native.o.thinlto.bc{{$}}
+OUTPUTS-DAG: _start.2.[[PID]].native.o.thinlto.bc{{$}}
+OUTPUTS-DAG:    bar.3.[[PID]].native.o.thinlto.bc{{$}}
+
+# Native output object files.
+OUTPUTS-DAG:    dog.1.[[PID]].native.o{{$}}
+OUTPUTS-DAG: _start.2.[[PID]].native.o{{$}}
+OUTPUTS-DAG:    bar.3.[[PID]].native.o{{$}}
+
+#--- foo.c
+__attribute__((retain)) void foo() {}
+
+#--- bar.c
+extern void foo();
+__attribute__((retain)) void bar() { foo(); }
+
+#--- dog.c
+__attribute__((retain)) void dog() {}
+
+#--- cat.c
+__attribute__((retain)) void cat() {}
+
+#--- _start.c
+extern void bar();
+__attribute__((retain)) void _start() {
+  bar();
+}
+

cross-project-tests/dtlto/dtlto.c

@@ -0,0 +1,35 @@
+// REQUIRES: x86-registered-target,ld.lld
+
+/// Simple test that DTLTO works with a single input bitcode file and that
+/// --save-temps can be applied to the remote compilation.
+// RUN: rm -rf %t && mkdir %t && cd %t
+
+// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s
+
+// RUN: ld.lld dtlto.o \
+// RUN:   --thinlto-distributor=%python \
+// RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+// RUN:   --thinlto-remote-compiler=%clang \
+// RUN:   --thinlto-remote-compiler-arg=--save-temps
+
+/// Check that the required output files have been created.
+// RUN: ls | count 10
+// RUN: ls | FileCheck %s
+
+/// Produced by the bitcode compilation.
+// CHECK-DAG: {{^}}dtlto.o{{$}}
+
+/// Linked ELF.
+// CHECK-DAG: {{^}}a.out{{$}}
+
+/// --save-temps output for the backend compilation.
+// CHECK-DAG: {{^}}dtlto.s{{$}}
+// CHECK-DAG: {{^}}dtlto.s.0.preopt.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.1.promote.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.2.internalize.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.3.import.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.4.opt.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.5.precodegen.bc{{$}}
+// CHECK-DAG: {{^}}dtlto.s.resolution.txt{{$}}
+
+int _start() { return 0; }

cross-project-tests/dtlto/lit.local.cfg

@@ -0,0 +1,2 @@
+if "clang" not in config.available_features:
+    config.unsupported = True

cross-project-tests/dtlto/path.test

@@ -0,0 +1,65 @@
+# REQUIRES: x86-registered-target,ld.lld,llvm-ar
+
+# Check that any shortened "8.3" paths containing '~' characters are expanded,
+# and their path seperators changed to Winodws style ones, prior to being passed
+# to SN-DBS during DTLTO.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+# Create an archive to confirm unpacked member paths are expanded.
+RUN: prospero-clang --target=x86_64-linux-gnu -c start.c f.c -flto=thin -O2
+RUN: prospero-llvm-ar rcs libf.a f.o --thin
+
+RUN: %python in-tilde-dir.py \
+RUN:   ld.lld start.o libf.a -o start.elf \
+RUN:     --thinlto-distributor=%python \
+RUN:     --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+RUN:     --thinlto-remote-compiler=%clang \
+RUN:     --save-temps
+
+# Ensure that cross-module importing occurred. Cross-module importing is somewhat
+# fragile - e.g. it requires amenable code and compilation with -O2 or above.
+# It's important that importing happens for this test case, as the paths to
+# imported-from modules are recorded in the ThinLTO metadata.
+RUN: prospero-llvm-objdump -d start.elf | tee d.txt | FileCheck %s --check-prefix=IMPORT
+IMPORT: <_start>:
+IMPORT: jmp 0x{{[0-9A-F]+}} <_start>
+IMPORT: <f>:
+IMPORT-NEXT: jmp 0x{{[0-9A-F]+}} <f>
+
+RUN: FileCheck --input-file start.*.dist-file.json %s --check-prefix=TILDE
+TILDE-NOT: ~
+
+#--- f.c
+int _start();
+int f() { return _start(); }
+
+#--- start.c
+int f();
+int _start() { return f(); }
+
+#--- in-tilde-dir.py
+import os, shutil, sys, uuid, subprocess
+from pathlib import Path
+
+temp = Path(os.environ['TEMP'])
+assert temp.is_dir()
+
+# Copy the CWD to a unique directory inside TEMP and determine its 8.3 form.
+# TEMP is likely to be on a drive that supports long+short paths.
+d = (temp / str(uuid.uuid4()) / 'veryverylong').resolve()
+d83 = d.parent / (d.name[:6] + '~1')
+d.parent.mkdir(parents=True)
+try:
+    shutil.copytree(Path.cwd(), d)
+
+    # Replace the arguments of the command that name files with equivalents in
+    # our temp directory, prefixed with the 8.3 form.
+    cmd = [a if not Path(a).is_file() else str(d83/a) \
+           for a in sys.argv[1:]]
+    print(cmd)
+
+    sys.exit(subprocess.run(cmd).returncode)
+
+finally:
+    shutil.rmtree(d.parent)

cross-project-tests/lit.cfg.py

@@ -19,7 +19,7 @@
 config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files.
-config.suffixes = [".c", ".cl", ".cpp", ".m"]
+config.suffixes = [".c", ".cl", ".cpp", ".m", ".test"]
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
@@ -107,6 +107,8 @@ def get_required_attr(config, attr_name):
 if lldb_path is not None:
     config.available_features.add("lldb")
 
+if llvm_config.use_llvm_tool("llvm-ar"):
+    config.available_features.add("llvm-ar")
 
 def configure_dexter_substitutions():
     """Configure substitutions for host platform and return list of dependencies"""

lld/Common/Filesystem.cpp

@@ -22,6 +22,12 @@
 #include <unistd.h>
 #endif
 #include <thread>
+#if defined(_WIN32)
+#include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/Windows/WindowsSupport.h"
+#include "llvm/Support/WindowsError.h"
+#include <io.h>
+#endif
 
 using namespace llvm;
 using namespace lld;
@@ -155,3 +161,15 @@ std::unique_ptr<raw_fd_ostream> lld::openLTOOutputFile(StringRef file) {
     return fs;
   return openFile(file);
 }
+
+std::error_code lld::dtltoNormalizePath(std::string &PathOut) {
+#if defined(_WIN32)
+  llvm::SmallString<128> Expanded;
+  if (auto EC = llvm::sys::windows::makeLong(PathOut, Expanded))
+    return EC;
+
+  PathOut.assign(Expanded.begin(), Expanded.end());
+#endif
+  (void)PathOut;
+  return std::error_code{};
+}

lld/ELF/Config.h

@@ -249,6 +249,10 @@ struct Config {
   llvm::SmallVector<llvm::StringRef, 0> searchPaths;
   llvm::SmallVector<llvm::StringRef, 0> symbolOrderingFile;
   llvm::SmallVector<llvm::StringRef, 0> thinLTOModulesToCompile;
+  llvm::StringRef dtltoDistributor;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
+  llvm::StringRef dtltoCompiler;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
   llvm::SmallVector<llvm::StringRef, 0> undefined;
   llvm::SmallVector<SymbolVersion, 0> dynamicList;
   llvm::SmallVector<uint8_t, 0> buildIdVector;

lld/ELF/Driver.cpp

@@ -1341,6 +1341,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
   ctx.arg.disableVerify = args.hasArg(OPT_disable_verify);
   ctx.arg.discard = getDiscard(args);
+  ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq);
+  ctx.arg.dtltoDistributorArgs =
+      args::getStrings(args, OPT_thinlto_distributor_arg);
+  ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler_eq);
+  ctx.arg.dtltoCompilerArgs = args::getStrings(args, OPT_thinlto_compiler_arg);
   ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
   ctx.arg.dynamicLinker = getDynamicLinker(ctx, args);
   ctx.arg.ehFrameHdr =

lld/ELF/InputFiles.cpp

@@ -17,9 +17,11 @@
 #include "SyntheticSections.h"
 #include "Target.h"
 #include "lld/Common/DWARF.h"
+#include "lld/Common/Filesystem.h"
 #include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/LTO/LTO.h"
+#include "llvm/Object/Archive.h"
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Support/ARMAttributeParser.h"
 #include "llvm/Support/ARMBuildAttributes.h"
@@ -1739,6 +1741,36 @@ static uint8_t getOsAbi(const Triple &t) {
   }
 }
 
+// For DTLTO, bitcode member names must be valid paths to files on disk.
+// For thin archives, resolve `memberPath` relative to the archive's location.
+// Returns true if adjusted; false otherwise. Non-thin archives are unsupported.
+static bool dtltoAdjustMemberPathIfThinArchive(Ctx &ctx, StringRef archivePath,
+                                               std::string &memberPath) {
+  assert(!archivePath.empty() && !ctx.arg.dtltoDistributor.empty());
+
+  // Check if the archive file is a thin archive by reading its header.
+  auto bufferOrErr =
+      MemoryBuffer::getFileSlice(archivePath, sizeof(ThinArchiveMagic) - 1, 0);
+  if (std::error_code ec = bufferOrErr.getError()) {
+    ErrAlways(ctx) << "cannot open " << archivePath << ": " << ec.message();
+    return false;
+  }
+
+  if (!bufferOrErr->get()->getBuffer().starts_with(ThinArchiveMagic))
+    return false;
+
+  SmallString<64> resolvedPath;
+  if (path::is_relative(memberPath)) {
+    resolvedPath = path::parent_path(archivePath);
+    path::append(resolvedPath, memberPath);
+    memberPath = resolvedPath.str();
+  }
+
+  lld::dtltoNormalizePath(memberPath);
+
+  return true;
+}
+
 BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName,
                          uint64_t offsetInArchive, bool lazy)
     : InputFile(ctx, BitcodeKind, mb) {
@@ -1749,17 +1781,25 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName,
   if (ctx.arg.thinLTOIndexOnly)
     path = replaceThinLTOSuffix(ctx, mb.getBufferIdentifier());
 
+  // SCE_PRIVATE: begin DTLTO
+  // TODO: Investigate if we can remove this code.
+  if (!ctx.arg.dtltoDistributor.empty())
+    lld::dtltoNormalizePath(path);
+
   // ThinLTO assumes that all MemoryBufferRefs given to it have a unique
   // name. If two archives define two members with the same name, this
   // causes a collision which result in only one of the objects being taken
   // into consideration at LTO time (which very likely causes undefined
   // symbols later in the link stage). So we append file offset to make
   // filename unique.
   StringSaver &ss = ctx.saver;
-  StringRef name = archiveName.empty()
-                       ? ss.save(path)
-                       : ss.save(archiveName + "(" + path::filename(path) +
-                                 " at " + utostr(offsetInArchive) + ")");
+  StringRef name =
+      (archiveName.empty() ||
+       (!ctx.arg.dtltoDistributor.empty() &&
+        dtltoAdjustMemberPathIfThinArchive(ctx, archiveName, path)))
+          ? ss.save(path)
+          : ss.save(archiveName + "(" + path::filename(path) + " at " +
+                    utostr(offsetInArchive) + ")");
   MemoryBufferRef mbref(mb.getBuffer(), name);
 
   obj = CHECK2(lto::InputFile::create(mbref), this);

lld/ELF/LTO.cpp

@@ -180,6 +180,13 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
         std::string(ctx.arg.thinLTOPrefixReplaceNew),
         std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
         ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
+  } else if (!ctx.arg.dtltoDistributor.empty()) {
+    backend = lto::createOutOfProcessThinBackend(
+        llvm::hardware_concurrency(ctx.arg.thinLTOJobs), onIndexWrite,
+        ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles,
+        ctx.arg.outputFile, ctx.arg.dtltoDistributor,
+        ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler,
+        ctx.arg.dtltoCompilerArgs, !ctx.arg.saveTempsArgs.empty());
   } else {
     backend = lto::createInProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),

lld/ELF/Options.td

@@ -710,7 +710,17 @@ def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
 def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
 def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
   HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">;
-
+def thinlto_distributor_eq: JJ<"thinlto-distributor=">,
+  HelpText<"Distributor to use for ThinLTO backend compilations. If specified, "
+  "ThinLTO backend compilations will be distributed.">;
+defm thinlto_distributor_arg: EEq<"thinlto-distributor-arg", "Arguments to "
+  "pass to the ThinLTO distributor">;
+def thinlto_compiler_eq: JJ<"thinlto-remote-compiler=">,
+  HelpText<"Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
+  "compilations">;
+defm thinlto_compiler_arg: EEq<"thinlto-remote-compiler-arg", "Compiler "
+  "arguments for the ThinLTO distributor to pass for ThinLTO backend "
+  "compilations">;
 defm fat_lto_objects: BB<"fat-lto-objects",
     "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.",
     "Ignore the .llvm.lto section in relocatable object files (default).">;