[Driver] Dump device code during SYCL offloading. (#11132)

This PR enables users to dump the device code generated during the SYCL
compilation into a user provided directory.
The new option added is : -fsycl-dump-device-code=/path/to/directory/

**Example usage:**
If somebody runs : `clang++ -fsycl
-fsycl-dump-device-code=/path/to/directory test.cpp`

the `*.spv` files generated during device compilation show up in
`/path/to/directory/`

Author: srividya-sundaram

clang/include/clang/Driver/Driver.h

@@ -79,6 +79,8 @@ class Driver {
 
   IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS;
 
+  bool DumpDeviceCode;
+
   enum DriverMode {
     GCCMode,
     GXXMode,
@@ -420,6 +422,7 @@ class Driver {
     return Dir.c_str();
   }
   void setInstalledDir(StringRef Value) { InstalledDir = std::string(Value); }
+  bool isDumpDeviceCodeEnabled() const { return DumpDeviceCode; }
 
   bool isSaveTempsEnabled() const { return SaveTemps != SaveTempsNone; }
   bool isSaveTempsObj() const { return SaveTemps == SaveTempsObj; }

clang/include/clang/Driver/Options.td

@@ -5532,6 +5532,9 @@ def regcall4 : Flag<["-"], "regcall4">, Group<m_Group>,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Set __regcall4 as a default calling convention to respect __regcall ABI v.4">,
   MarshallingInfoFlag<LangOpts<"RegCall4">>;
+def fsycl_dump_device_code_EQ : Joined<["-"], "fsycl-dump-device-code=">, Flags<[NoXarchOption]>,
+  Visibility<[ClangOption, CLOption]>,
+  HelpText<"Dump device code into the user provided directory.">;
 def save_temps_EQ : Joined<["-", "--"], "save-temps=">, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Save intermediate compilation results.">;

clang/lib/Driver/Compilation.cpp

@@ -21,6 +21,7 @@
 #include "llvm/Option/OptSpecifier.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/SimpleTable.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
@@ -126,6 +127,20 @@ bool Compilation::CleanupFile(const char *File, bool IssueErrors) const {
   // Don't try to remove files which we don't have write access to (but may be
   // able to remove), or non-regular files. Underlying tools may have
   // intentionally not overwritten them.
+
+  // Save the device code files(spv files) only if -fsycl-dump-device-code
+  // option is enabled.
+  if (TheDriver.isDumpDeviceCodeEnabled()) {
+    Arg *DumpDeviceCodeArg =
+        getArgs().getLastArg(options::OPT_fsycl_dump_device_code_EQ);
+    std::string ExpectedDir =
+        DumpDeviceCodeArg ? DumpDeviceCodeArg->getValue() : "";
+    std::string ActualFile(File);
+    if (ActualFile.find(ExpectedDir) != std::string::npos &&
+        llvm::sys::path::extension(ActualFile).equals(".spv"))
+      return false;
+  }
+
   if (!llvm::sys::fs::can_write(File) || !llvm::sys::fs::is_regular_file(File))
     return true;
 
@@ -145,7 +160,7 @@ bool Compilation::CleanupFile(const char *File, bool IssueErrors) const {
 bool Compilation::CleanupFileList(const TempFileList &Files,
                                   bool IssueErrors) const {
   bool Success = true;
-  for (const auto &File: Files) {
+  for (const auto &File : Files) {
     // Temporary file lists contain files that need to be cleaned. The
     // file containing the information is also removed
     if (File.second == types::TY_Tempfilelist ||
@@ -161,6 +176,7 @@ bool Compilation::CleanupFileList(const TempFileList &Files,
             Success = false;
             continue;
           }
+
           std::vector<std::string> TmpFileNames;
           T->get()->linearize(TmpFileNames);
 
@@ -172,7 +188,7 @@ bool Compilation::CleanupFileList(const TempFileList &Files,
       } else {
         std::ifstream ListFile(File.first);
         std::string TmpFileName;
-        while (std::getline(ListFile, TmpFileName) && !TmpFileName.empty())
+        while (std::getline(ListFile, TmpFileName))
           Success &= CleanupFile(TmpFileName.c_str(), IssueErrors);
       }
     }

clang/lib/Driver/Driver.cpp

@@ -199,7 +199,7 @@ Driver::Driver(StringRef ClangExecutable, StringRef TargetTriple,
                DiagnosticsEngine &Diags, std::string Title,
                IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS)
     : Diags(Diags), VFS(std::move(VFS)), Mode(GCCMode),
-      SaveTemps(SaveTempsNone), BitcodeEmbed(EmbedNone),
+      SaveTemps(SaveTempsNone), DumpDeviceCode(false), BitcodeEmbed(EmbedNone),
       Offload(OffloadHostDevice), CXX20HeaderType(HeaderMode_None),
       ModulesModeCXX20(false), LTOMode(LTOK_None), OffloadLTOMode(LTOK_None),
       ClangExecutable(ClangExecutable), SysRoot(DEFAULT_SYSROOT),
@@ -1705,6 +1705,9 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
                     .Default(SaveTempsCwd);
   }
 
+  if (Args.getLastArg(options::OPT_fsycl_dump_device_code_EQ))
+    DumpDeviceCode = true;
+
   if (const Arg *A = Args.getLastArg(options::OPT_offload_host_only,
                                      options::OPT_offload_device_only,
                                      options::OPT_offload_host_device)) {

clang/lib/Driver/ToolChains/Clang.cpp

@@ -10184,6 +10184,26 @@ void SPIRVTranslator::ConstructJob(Compilation &C, const JobAction &JA,
         TCArgs.MakeArgString("--out-file-list=" + OutputFileName));
     ForeachArgs.push_back(
         TCArgs.MakeArgString("--out-replace=" + OutputFileName));
+    // If fsycl-dump-device-code is passed, put the output files from llvm-spirv
+    // into the path provided in fsycl-dump-device-code.
+    if (C.getDriver().isDumpDeviceCodeEnabled()) {
+      SmallString<128> OutputDir;
+
+      Arg *DumpDeviceCodeArg =
+          C.getArgs().getLastArg(options::OPT_fsycl_dump_device_code_EQ);
+
+      OutputDir = (DumpDeviceCodeArg ? DumpDeviceCodeArg->getValue() : "");
+
+      // If the output directory path is empty, put the llvm-spirv output in the
+      // current directory.
+      if (OutputDir.empty())
+        llvm::sys::path::native(OutputDir = "./");
+      else
+        OutputDir.append(llvm::sys::path::get_separator());
+      ForeachArgs.push_back(
+          C.getArgs().MakeArgString("--out-dir=" + OutputDir));
+    }
+
     StringRef ParallelJobs =
         TCArgs.getLastArgValue(options::OPT_fsycl_max_parallel_jobs_EQ);
     if (!ParallelJobs.empty())

clang/test/Driver/sycl-dump-device-code.cpp

@@ -0,0 +1,35 @@
+// SYCL offloading tests using -fsycl-dump-device-code
+
+// Verify that -fsycl-dump-device-code puts the device code (.spv files)
+// in the user provided directory.
+
+// Linux
+// clang -fsycl -target x86_64-unknown-linux-gnu
+// RUN: %clang -fsycl -fno-sycl-instrument-device-code -fno-sycl-device-lib=all -target x86_64-unknown-linux-gnu -fsycl-dump-device-code=/user/input/path %s -### 2>&1 \
+// RUN: | FileCheck %s --check-prefixes=CHK-FSYCL-DUMP-DEVICE-CODE
+
+// clang -fsycl -fsycl-targets=spir64-unknown-unknown
+// RUN: %clang -fsycl  -fsycl-targets=spir64-unknown-unknown -target x86_64-unknown-linux-gnu -fsycl-dump-device-code=/user/input/path %s -### 2>&1 \
+// RUN: | FileCheck %s --check-prefixes=CHK-FSYCL-DUMP-DEVICE-CODE
+
+// clang --driver-mode=g++
+// RUN: %clangxx -fsycl  -fsycl-targets=spir64-unknown-unknown -target x86_64-unknown-linux-gnu -fsycl-dump-device-code=/user/input/path %s -### 2>&1 \
+// RUN: | FileCheck %s --check-prefixes=CHK-FSYCL-DUMP-DEVICE-CODE
+
+// Windows
+// RUN: %clang_cl -fsycl -fsycl-dump-device-code=/user/input/path %s -### 2>&1 \
+// RUN: | FileCheck %s --check-prefixes=CHK-FSYCL-DUMP-DEVICE-CODE-WIN
+
+// CHK-FSYCL-DUMP-DEVICE-CODE: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv"
+// CHK-FSYCL-DUMP-DEVICE-CODE-WIN: llvm-foreach{{.*}} "--out-dir=/user/input/path{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv"
+
+// Linux
+// RUN: %clang -fsycl  -fsycl-targets=spir64-unknown-unknown -target x86_64-unknown-linux-gnu -fsycl-dump-device-code= %s -### 2>&1 \
+// RUN: | FileCheck %s --check-prefixes=CHK-FSYCL-DUMP-DEVICE-CODE-CWD
+
+// Windows
+// RUN: %clang_cl -fsycl -fsycl-dump-device-code= %s -### 2>&1 \
+// RUN: | FileCheck %s --check-prefixes=CHK-FSYCL-DUMP-DEVICE-CODE-WIN-CWD
+
+// CHK-FSYCL-DUMP-DEVICE-CODE-CWD: llvm-foreach{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv"
+// CHK-FSYCL-DUMP-DEVICE-CODE-WIN-CWD: llvm-foreach{{.*}} "--out-dir=.{{(/|\\\\)}}" "--" "{{.*}}llvm-spirv"