[DTLTO] Normalize Windows 8.3 short paths to long form

8.3-style short paths (e.g., C:\PROGRA~1) can cause inconsistencies or
failures on remote machines during distributed linking. Such paths may
not resolve correctly or may differ between systems. We anticipate
that many/all distribution systems will have difficulties with such
paths.

This patch ensures that all Windows paths used for DTLTO in ELF LLD are
converted to their long form using GetLongPathNameW. This avoids
ambiguity and improves portability across most distribution systems.

Author: bd1976bris

cross-project-tests/dtlto/path.test

@@ -0,0 +1,65 @@
+# REQUIRES: x86-registered-target,ld.lld,llvm-ar
+
+# Check that any shortened "8.3" paths containing '~' characters are expanded,
+# and their path seperators changed to Winodws style ones, prior to being passed
+# to SN-DBS during DTLTO.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+# Create an archive to confirm unpacked member paths are expanded.
+RUN: prospero-clang --target=x86_64-linux-gnu -c start.c f.c -flto=thin -O2
+RUN: prospero-llvm-ar rcs libf.a f.o --thin
+
+RUN: %python in-tilde-dir.py \
+RUN:   ld.lld start.o libf.a -o start.elf \
+RUN:     --thinlto-distributor=%python \
+RUN:     --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+RUN:     --thinlto-remote-compiler=%clang \
+RUN:     --save-temps
+
+# Ensure that cross-module importing occurred. Cross-module importing is somewhat
+# fragile - e.g. it requires amenable code and compilation with -O2 or above.
+# It's important that importing happens for this test case, as the paths to
+# imported-from modules are recorded in the ThinLTO metadata.
+RUN: prospero-llvm-objdump -d start.elf | tee d.txt | FileCheck %s --check-prefix=IMPORT
+IMPORT: <_start>:
+IMPORT: jmp 0x{{[0-9A-F]+}} <_start>
+IMPORT: <f>:
+IMPORT-NEXT: jmp 0x{{[0-9A-F]+}} <f>
+
+RUN: FileCheck --input-file start.*.dist-file.json %s --check-prefix=TILDE
+TILDE-NOT: ~
+
+#--- f.c
+int _start();
+int f() { return _start(); }
+
+#--- start.c
+int f();
+int _start() { return f(); }
+
+#--- in-tilde-dir.py
+import os, shutil, sys, uuid, subprocess
+from pathlib import Path
+
+temp = Path(os.environ['TEMP'])
+assert temp.is_dir()
+
+# Copy the CWD to a unique directory inside TEMP and determine its 8.3 form.
+# TEMP is likely to be on a drive that supports long+short paths.
+d = (temp / str(uuid.uuid4()) / 'veryverylong').resolve()
+d83 = d.parent / (d.name[:6] + '~1')
+d.parent.mkdir(parents=True)
+try:
+    shutil.copytree(Path.cwd(), d)
+
+    # Replace the arguments of the command that name files with equivalents in
+    # our temp directory, prefixed with the 8.3 form.
+    cmd = [a if not Path(a).is_file() else str(d83/a) \
+           for a in sys.argv[1:]]
+    print(cmd)
+
+    sys.exit(subprocess.run(cmd).returncode)
+
+finally:
+    shutil.rmtree(d.parent)

lld/Common/Filesystem.cpp

@@ -22,6 +22,12 @@
 #include <unistd.h>
 #endif
 #include <thread>
+#if defined(_WIN32)
+#include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/Windows/WindowsSupport.h"
+#include "llvm/Support/WindowsError.h"
+#include <io.h>
+#endif
 
 using namespace llvm;
 using namespace lld;
@@ -155,3 +161,15 @@ std::unique_ptr<raw_fd_ostream> lld::openLTOOutputFile(StringRef file) {
     return fs;
   return openFile(file);
 }
+
+std::error_code lld::dtltoNormalizePath(std::string &PathOut) {
+#if defined(_WIN32)
+  llvm::SmallString<128> Expanded;
+  if (auto EC = llvm::sys::windows::makeLong(PathOut, Expanded))
+    return EC;
+
+  PathOut.assign(Expanded.begin(), Expanded.end());
+#endif
+  (void)PathOut;
+  return std::error_code{};
+}

lld/ELF/InputFiles.cpp

@@ -17,6 +17,7 @@
 #include "SyntheticSections.h"
 #include "Target.h"
 #include "lld/Common/DWARF.h"
+#include "lld/Common/Filesystem.h"
 #include "llvm/ADT/CachedHashString.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/LTO/LTO.h"
@@ -1764,6 +1765,9 @@ static bool dtltoAdjustMemberPathIfThinArchive(Ctx &ctx, StringRef archivePath,
     path::append(resolvedPath, memberPath);
     memberPath = resolvedPath.str();
   }
+
+  lld::dtltoNormalizePath(memberPath);
+
   return true;
 }
 
@@ -1777,6 +1781,11 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName,
   if (ctx.arg.thinLTOIndexOnly)
     path = replaceThinLTOSuffix(ctx, mb.getBufferIdentifier());
 
+  // SCE_PRIVATE: begin DTLTO
+  // TODO: Investigate if we can remove this code.
+  if (!ctx.arg.dtltoDistributor.empty())
+    lld::dtltoNormalizePath(path);
+
   // ThinLTO assumes that all MemoryBufferRefs given to it have a unique
   // name. If two archives define two members with the same name, this
   // causes a collision which result in only one of the objects being taken

lld/include/lld/Common/Filesystem.h

@@ -19,6 +19,7 @@ void unlinkAsync(StringRef path);
 std::error_code tryCreateFile(StringRef path);
 std::unique_ptr<llvm::raw_fd_ostream> openFile(StringRef file);
 std::unique_ptr<llvm::raw_fd_ostream> openLTOOutputFile(StringRef file);
+std::error_code dtltoNormalizePath(std::string &PathOut);
 } // namespace lld
 
 #endif

llvm/include/llvm/Support/Windows/WindowsSupport.h

@@ -245,6 +245,14 @@ LLVM_ABI std::error_code widenPath(const Twine &Path8,
                                    SmallVectorImpl<wchar_t> &Path16,
                                    size_t MaxPathLen = MAX_PATH);
 
+/// Convert UTF-8 path into a long form UTF-8 path.
+LLVM_ABI std::error_code makeLong(const Twine &Path8,
+                                  llvm::SmallVectorImpl<char> &Result);
+
+/// Retrieves the volume mount point from UTF-16 path as UTF-16 result.
+std::error_code getVolumeRootFromPath(SmallVectorImpl<wchar_t> &Path,
+                                      SmallVectorImpl<wchar_t> &Result);
+
 } // end namespace windows
 } // end namespace sys
 } // end namespace llvm.

llvm/lib/Support/Windows/Path.inc

@@ -58,6 +58,19 @@ static bool is_separator(const wchar_t value) {
   }
 }
 
+static void stripExtendedPrefix(wchar_t *&Data, DWORD &CountChars) {
+  if (CountChars >= 8 && ::memcmp(Data, L"\\\\?\\UNC\\", 16) == 0) {
+    // Convert \\?\UNC\foo\bar to \\foo\bar
+    CountChars -= 6;
+    Data += 6;
+    Data[0] = '\\';
+  } else if (CountChars >= 4 && ::memcmp(Data, L"\\\\?\\", 8) == 0) {
+    // Convert \\?\C:\foo to C:\foo
+    CountChars -= 4;
+    Data += 4;
+  }
+}
+
 namespace llvm {
 namespace sys {
 namespace windows {
@@ -125,6 +138,70 @@ std::error_code widenPath(const Twine &Path8, SmallVectorImpl<wchar_t> &Path16,
   return UTF8ToUTF16(FullPath, Path16);
 }
 
+std::error_code makeLong(const Twine &Path8,
+                         llvm::SmallVectorImpl<char> &Result8) {
+  constexpr StringLiteral LongPathPrefix = R"(\\?\)";
+
+  SmallString<128> PathStorage;
+  StringRef PathStr = Path8.toStringRef(PathStorage);
+  bool HadPrefix = PathStr.starts_with(LongPathPrefix);
+
+  // Convert UTF-8 to UTF-16
+  SmallVector<wchar_t, 128> Path16;
+  if (std::error_code EC = widenPath(PathStr, Path16))
+    return EC;
+
+  // Start with a buffer equal to input.
+  llvm::SmallVector<wchar_t, 128> Long16;
+  DWORD Len = static_cast<DWORD>(Path16.size());
+
+  do {
+    Long16.resize_for_overwrite(Len); // grow
+
+    Len = ::GetLongPathNameW(Path16.data(), Long16.data(), Len);
+
+    // A zero return value indicates a failure other than insufficient space.
+    if (Len == 0)
+      return mapWindowsError(::GetLastError());
+
+    // If there's insufficient space, the len returned is larger than the len
+    // given - in this case len includes the null-terminator.
+  } while (Len >= Long16.size());
+
+  // On success, GetLongPathNameW returns the number of characters not
+  // including the null-terminator.
+  Long16.truncate(Len);
+  // Strip \\?\ or \\?\UNC\ prefix if it wasn't part of the original path
+  wchar_t *Data = Long16.data();
+  if (!HadPrefix)
+    stripExtendedPrefix(Data, Len);
+
+  return sys::windows::UTF16ToUTF8(Data, Len, Result8);
+}
+
+std::error_code getVolumeRootFromPath(SmallVectorImpl<wchar_t> &Path,
+                                      SmallVectorImpl<wchar_t> &VolumeRoot) {
+  size_t Len = 128;
+  while (true) {
+    VolumeRoot.resize(Len);
+    BOOL Success =
+        ::GetVolumePathNameW(Path.data(), VolumeRoot.data(), VolumeRoot.size());
+    if (Success)
+      break;
+
+    DWORD Err = ::GetLastError();
+    if (Err != ERROR_INSUFFICIENT_BUFFER)
+      return mapWindowsError(Err);
+
+    Len *= 2;
+  }
+
+  // Ensure null-termination in edge case where buffer was exactly full
+  VolumeRoot.push_back(L'\0');
+  VolumeRoot.truncate(wcslen(VolumeRoot.data()));
+  return std::error_code();
+}
+
 } // end namespace windows
 
 namespace fs {
@@ -309,29 +386,10 @@ std::error_code remove(const Twine &path, bool IgnoreNonExisting) {
 static std::error_code is_local_internal(SmallVectorImpl<wchar_t> &Path,
                                          bool &Result) {
   SmallVector<wchar_t, 128> VolumePath;
-  size_t Len = 128;
-  while (true) {
-    VolumePath.resize(Len);
-    BOOL Success =
-        ::GetVolumePathNameW(Path.data(), VolumePath.data(), VolumePath.size());
-
-    if (Success)
-      break;
-
-    DWORD Err = ::GetLastError();
-    if (Err != ERROR_INSUFFICIENT_BUFFER)
-      return mapWindowsError(Err);
+  if (std::error_code EC = windows::getVolumeRootFromPath(Path, VolumePath))
+    return EC;
 
-    Len *= 2;
-  }
-  // If the output buffer has exactly enough space for the path name, but not
-  // the null terminator, it will leave the output unterminated.  Push a null
-  // terminator onto the end to ensure that this never happens.
-  VolumePath.push_back(L'\0');
-  VolumePath.truncate(wcslen(VolumePath.data()));
-  const wchar_t *P = VolumePath.data();
-
-  UINT Type = ::GetDriveTypeW(P);
+  UINT Type = ::GetDriveTypeW(VolumePath.data());
   switch (Type) {
   case DRIVE_FIXED:
     Result = true;
@@ -392,16 +450,7 @@ static std::error_code realPathFromHandle(HANDLE H,
   // paths don't get canonicalized by file APIs.
   wchar_t *Data = Buffer.data();
   DWORD CountChars = Buffer.size();
-  if (CountChars >= 8 && ::memcmp(Data, L"\\\\?\\UNC\\", 16) == 0) {
-    // Convert \\?\UNC\foo\bar to \\foo\bar
-    CountChars -= 6;
-    Data += 6;
-    Data[0] = '\\';
-  } else if (CountChars >= 4 && ::memcmp(Data, L"\\\\?\\", 8) == 0) {
-    // Convert \\?\c:\foo to c:\foo
-    CountChars -= 4;
-    Data += 4;
-  }
+  stripExtendedPrefix(Data, CountChars);
 
   // Convert the result from UTF-16 to UTF-8.
   if (std::error_code EC = UTF16ToUTF8(Data, CountChars, RealPath))