[DO NOT MERGE][SYCLomatic] reMigration

clang/CMakeLists.txt

@@ -911,6 +911,7 @@ if (NOT CLANG_BUILT_STANDALONE)
     list(APPEND DPCT_RUN
       install-dpct-intercept-build
       install-dpct-pattern-rewriter
+      install-dpct-gitdiff2yaml
       install-dpct-headers
       install-clang-resource-headers
       install-dpct-binary

clang/include/clang/DPCT/DPCTOptions.inc

@@ -958,6 +958,12 @@ DPCT_FLAG_OPTION(
                    "--out-root directory. Default: off."),
     llvm::cl::cat(CtHelpCatAll), llvm::cl::cat(CtHelpCatCodeGen))
 
+DPCT_FLAG_OPTION(
+    ReMigration, clang::dpct::DpctOptionClass::OC_Attribute,
+    DPCT_OPTION_ACTIONS(clang::dpct::DpctActionKind::DAK_Migration),
+    "remigration", llvm::cl::desc("Enable remigration. Default: off."),
+    llvm::cl::cat(CtHelpCatAll), llvm::cl::cat(CtHelpCatCodeGen))
+
 DPCT_FLAG_OPTION(
     PathToHelperFunction, clang::dpct::DpctOptionClass::OC_Action,
     DPCT_OPTION_ACTIONS(clang::dpct::DpctActionKind::DAK_Help),

clang/lib/DPCT/AnalysisInfo.cpp

@@ -2513,6 +2513,7 @@ unsigned DpctGlobalInfo::ExperimentalFlag = 0;
 unsigned DpctGlobalInfo::HelperFuncPreferenceFlag = 0;
 bool DpctGlobalInfo::AnalysisModeFlag = false;
 bool DpctGlobalInfo::UseSYCLCompatFlag = false;
+bool DpctGlobalInfo::ReMigrationFlag = false;
 bool DpctGlobalInfo::CVersionCUDALaunchUsedFlag = false;
 unsigned int DpctGlobalInfo::ColorOption = 1;
 std::unordered_map<int, std::shared_ptr<DeviceFunctionInfo>>

clang/lib/DPCT/AnalysisInfo.h

@@ -1349,6 +1349,8 @@ class DpctGlobalInfo {
   static bool isCVersionCUDALaunchUsed() { return CVersionCUDALaunchUsedFlag; }
   static void setUseSYCLCompat(bool Flag = true) { UseSYCLCompatFlag = Flag; }
   static bool useSYCLCompat() { return UseSYCLCompatFlag; }
+  static void setReMigration(bool Flag = true) { ReMigrationFlag = Flag; }
+  static bool useReMigration() { return ReMigrationFlag; }
   static bool useEnqueueBarrier() {
     return getUsingExtensionDE(
         DPCPPExtensionsDefaultEnabled::ExtDE_EnqueueBarrier);
@@ -1682,6 +1684,7 @@ class DpctGlobalInfo {
   static unsigned HelperFuncPreferenceFlag;
   static bool AnalysisModeFlag;
   static bool UseSYCLCompatFlag;
+  static bool ReMigrationFlag;
   static bool CVersionCUDALaunchUsedFlag;
   static unsigned int ColorOption;
   static std::unordered_map<int, std::shared_ptr<DeviceFunctionInfo>>

clang/lib/DPCT/CMakeLists.txt

@@ -233,6 +233,7 @@ add_clang_library(DPCT
   MigrateScript/GenMakefile.cpp
   RulesInclude/InclusionHeaders.cpp
   IncMigration/IncrementalMigrationUtility.cpp
+  IncMigration/ReMigration.cpp
   UserDefinedRules/UserDefinedRules.cpp
   UserDefinedRules/PatternRewriter.cpp
   MigrateScript/MigrateBuildScript.cpp

clang/lib/DPCT/DPCT.cpp

@@ -79,7 +79,7 @@ using namespace clang::tooling;
 using namespace llvm::cl;
 
 extern bool isDPCT;
-
+extern bool ReMigrationReady;
 namespace clang {
 namespace tooling {
 UnifiedPath getFormatSearchPath();
@@ -516,7 +516,7 @@ static void loadMainSrcFileInfo(clang::tooling::UnifiedPath YamlFilePath) {
   if (!llvm::sys::fs::exists(YamlFilePath.getCanonicalPath()))
     return;
   auto PreTU = std::make_shared<clang::tooling::TranslationUnitReplacements>();
-  if (loadFromYaml(YamlFilePath, *PreTU) != 0) {
+  if (loadTUFromYaml(YamlFilePath, *PreTU) != 0) {
     llvm::errs() << getLoadYamlFailWarning();
   }
   DpctGlobalInfo::setMainSourceYamlTUR(PreTU);
@@ -804,6 +804,7 @@ int runDPCT(int argc, const char **argv) {
   CudaIncludePath = CudaInclude;
   SDKPath = SDKPathOpt;
 
+  DpctGlobalInfo::setReMigration(ReMigration);
   loadMainSrcFileInfo(OutRootPath.getCanonicalPath() + "/MainSourceFiles.yaml");
 
   std::transform(
@@ -1391,6 +1392,8 @@ int runDPCT(int argc, const char **argv) {
     dpctExit(MigrationSucceeded, false);
   }
 
+  ReMigrationReady = tryLoadingUpstreamChangesAndUserChanges();
+
   ReplTy ReplCUDA, ReplSYCL;
   volatile int RunCount = 0;
   do {

clang/lib/DPCT/FileGenerator/GenFiles.cpp

@@ -48,6 +48,8 @@ extern DpctOption<clang::dpct::opt, bool> ProcessAll;
 extern DpctOption<dpct::opt, std::string> BuildScriptFile;
 extern DpctOption<dpct::opt, bool> GenBuildScript;
 extern std::map<std::string, uint64_t> ErrorCnt;
+bool ReMigrationReady = false;
+extern std::set<std::string> MainSrcFilesHasCudaSyntex;
 
 namespace clang {
 namespace tooling {
@@ -188,18 +190,10 @@ bool rewriteCanonicalDir(clang::tooling::UnifiedPath &FilePath,
   return Result;
 }
 
-void rewriteFileName(clang::tooling::UnifiedPath &FileName) {
-  rewriteFileName(FileName, FileName);
-}
-
-void rewriteFileName(clang::tooling::UnifiedPath &FileName,
-                     const clang::tooling::UnifiedPath &FullPathName) {
-  std::string FilePath = FileName.getPath().str();
-  rewriteFileName(FilePath, FullPathName.getPath().str());
-  FileName = FilePath;
-}
-
-void rewriteFileName(std::string &FileName, const std::string &FullPathName) {
+static void
+rewriteFileName(std::string &FileName, const std::string &FullPathName,
+                std::function<bool(tooling::UnifiedPath)> HasCUDASyntax =
+                    DpctGlobalInfo::hasCUDASyntax) {
   SmallString<512> CanonicalPathStr(FullPathName);
   const auto Extension = path::extension(CanonicalPathStr);
   SourceProcessType FileType = GetSourceFileType(FullPathName);
@@ -210,8 +204,7 @@ void rewriteFileName(std::string &FileName, const std::string &FullPathName) {
     if (FileType & SPT_CudaSource) {
       path::replace_extension(CanonicalPathStr,
                               DpctGlobalInfo::getSYCLSourceExtension());
-    } else if ((FileType & SPT_CppSource) &&
-               DpctGlobalInfo::hasCUDASyntax(FileName)) {
+    } else if ((FileType & SPT_CppSource) && HasCUDASyntax(FileName)) {
       path::replace_extension(CanonicalPathStr,
                               Extension +
                                   DpctGlobalInfo::getSYCLSourceExtension());
@@ -224,6 +217,17 @@ void rewriteFileName(std::string &FileName, const std::string &FullPathName) {
   FileName = CanonicalPathStr.c_str();
 }
 
+void rewriteFileName(clang::tooling::UnifiedPath &FileName,
+                     const clang::tooling::UnifiedPath &FullPathName) {
+  std::string FilePath = FileName.getPath().str();
+  rewriteFileName(FilePath, FullPathName.getPath().str());
+  FileName = FilePath;
+}
+
+void rewriteFileName(clang::tooling::UnifiedPath &FileName) {
+  rewriteFileName(FileName, FileName);
+}
+
 static std::vector<std::string> FilesNotInCompilationDB;
 
 std::map<std::string, std::string> OutFilePath2InFilePath;
@@ -506,7 +510,7 @@ static void getReplsFromTUR(const std::string &FilePath,
 }
 
 int writeReplacementsToFiles(
-    ReplTy &Replset, Rewriter &Rewrite, const std::string &Folder,
+    ReplTy &Replset2, Rewriter &Rewrite, const std::string &Folder,
     clang::tooling::UnifiedPath &InRoot,
     std::unordered_map<std::string, bool> &MainSrcFileMap,
     std::vector<clang::tooling::Replacement> &AllFilesRepls,
@@ -520,6 +524,48 @@ int writeReplacementsToFiles(
   volatile ProcessStatus status = MigrationSucceeded;
   clang::tooling::UnifiedPath OutPath;
 
+  ReplTy Replset;
+  if (ReMigrationReady) {
+    std::vector<clang::tooling::Replacement> Repl_B;
+    std::vector<clang::tooling::Replacement> Repl_C1;
+    for (const auto &Entry : Replset2) {
+      for (const auto &Repl : Entry.second) {
+        Repl_B.push_back(Repl);
+      }
+    }
+    for (const auto &Repl : clang::dpct::getLastMigration()->Replacements) {
+      Repl_C1.push_back(Repl);
+    }
+
+    std::map<std::string /*SYCL name*/, std::string /*CUDA name*/> FileNameMap;
+    auto hasCUDASyntax = [](tooling::UnifiedPath Path) -> bool {
+      if (MainSrcFilesHasCudaSyntex.find(Path.getCanonicalPath().str()) !=
+          MainSrcFilesHasCudaSyntex.end())
+        return true;
+      return false;
+    };
+    for (const auto &Entry : Repl_C1) {
+      std::string CUDAFilePath = Entry.getFilePath().str();
+      std::string SYCLFilePath;
+      rewriteFileName(SYCLFilePath, CUDAFilePath, hasCUDASyntax);
+      FileNameMap[SYCLFilePath] = CUDAFilePath;
+    }
+    std::map<std::string, std::vector<clang::tooling::Replacement>> Result =
+        clang::dpct::reMigrationMerge(clang::dpct::getUpstreamChanges(), Repl_B,
+                                      Repl_C1, clang::dpct::getUserChanges(),
+                                      FileNameMap);
+    for (const auto &Entry : Result) {
+      clang::tooling::Replacements Repls;
+      for (const auto &Repl : Entry.second) {
+        llvm::cantFail(Repls.add(Repl));
+      }
+      Replset.insert(std::make_pair(Entry.first, Repls));
+    }
+  } else {
+    // If not re-migration, just use the original Replset.
+    Replset = Replset2;
+  }
+
   for (auto &Entry : Replset) {
     OutPath = StringRef(DpctGlobalInfo::removeSymlinks(
         Rewrite.getSourceMgr().getFileManager(), Entry.first));
@@ -633,6 +679,7 @@ int writeReplacementsToFiles(
     // We have written a migrated file; Update the output file path info
     OutFilePath2InFilePath[OutPath.getCanonicalPath().str()] = Entry.first;
   }
+  Replset2 = Replset;
   return status;
 }
 

clang/lib/DPCT/FileGenerator/GenFiles.h

@@ -76,9 +76,6 @@ void rewriteFileName(clang::tooling::UnifiedPath &FileName);
 void rewriteFileName(clang::tooling::UnifiedPath &FileName,
                      const clang::tooling::UnifiedPath &FullPathName);
 
-// Replace file name \p FileName with new migrated name.
-void rewriteFileName(std::string &FileName, const std::string &FullPathName);
-
 // A mapping from output file path to it's corresponding input file.
 extern std::map<std::string, std::string> OutFilePath2InFilePath;
 

clang/lib/DPCT/IncMigration/ExternalReplacement.cpp

@@ -10,19 +10,19 @@
 //   -Load replacement from external (disk file)
 //   -Merge replacement in current migration with previous migration.
 
+#include "ExternalReplacement.h"
 #include "AnalysisInfo.h"
+#include "IncMigration/IncrementalMigrationUtility.h"
 #include "Utility.h"
+
+#include "clang/Tooling/Core/Diagnostic.h"
 #include "clang/Tooling/Core/Replacement.h"
+#include "clang/Tooling/Refactoring.h"
+#include "clang/Tooling/ReplacementsYaml.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
-
-#include "ExternalReplacement.h"
-#include "IncMigration/IncrementalMigrationUtility.h"
-#include "clang/Tooling/Core/Diagnostic.h"
-#include "clang/Tooling/Refactoring.h"
-#include "clang/Tooling/ReplacementsYaml.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_os_ostream.h"
 
@@ -77,18 +77,29 @@ int save2Yaml(
   return 0;
 }
 
-int loadFromYaml(const clang::tooling::UnifiedPath &Input,
-                 clang::tooling::TranslationUnitReplacements &TU) {
+template <class T>
+static int loadFromYaml(const clang::tooling::UnifiedPath &Input, T &Content) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
       llvm::MemoryBuffer::getFile(Input.getCanonicalPath());
   if (!Buffer) {
     llvm::errs() << "error: failed to read " << Input.getCanonicalPath() << ": "
                  << Buffer.getError().message() << "\n";
     return -1;
   }
-
   llvm::yaml::Input YAMLIn(Buffer.get()->getBuffer());
-  YAMLIn >> TU;
+  YAMLIn >> Content;
+  if (YAMLIn.error()) {
+    Content = T();
+    return -1;
+  }
+  return 0;
+}
+
+int loadTUFromYaml(const clang::tooling::UnifiedPath &Input,
+                   clang::tooling::TranslationUnitReplacements &TU) {
+  int Status = loadFromYaml(Input, TU);
+  if (Status)
+    return Status;
 
   bool IsSrcFileChanged = false;
   for (const auto &digest : TU.MainSourceFilesDigest) {
@@ -100,7 +111,7 @@ int loadFromYaml(const clang::tooling::UnifiedPath &Input,
     }
   }
 
-  if (IsSrcFileChanged || YAMLIn.error()) {
+  if (IsSrcFileChanged && !DpctGlobalInfo::useReMigration()) {
     // File doesn't appear to be a header change description. Ignore it.
     TU = clang::tooling::TranslationUnitReplacements();
     return -1;
@@ -109,6 +120,16 @@ int loadFromYaml(const clang::tooling::UnifiedPath &Input,
   return 0;
 }
 
+void loadGDCFromYaml(const clang::tooling::UnifiedPath &Input,
+                     clang::dpct::GitDiffChanges &GDC) {
+  int status = loadFromYaml(Input, GDC);
+  if (status) {
+    llvm::errs() << "Failed to load git diff Changes from "
+                 << Input.getCanonicalPath() << "\n";
+    GDC = clang::dpct::GitDiffChanges();
+  }
+}
+
 void mergeAndUniqueReps(
     Replacements &Replaces,
     const std::vector<clang::tooling::Replacement> &PreRepls) {

clang/lib/DPCT/IncMigration/ExternalReplacement.h

@@ -9,8 +9,8 @@
 #ifndef __EXTERNAL_REPLACEMENT_H__
 #define __EXTERNAL_REPLACEMENT_H__
 
-#include "clang/Tooling/Core/Replacement.h"
-#include "llvm/ADT/StringRef.h"
+#include "IncMigration/ReMigration.h"
+
 #include <map>
 #include <vector>
 
@@ -27,8 +27,10 @@ class Replacements;
 
 namespace clang {
 namespace dpct {
-int loadFromYaml(const clang::tooling::UnifiedPath &Input,
-                 clang::tooling::TranslationUnitReplacements &TU);
+int loadTUFromYaml(const clang::tooling::UnifiedPath &Input,
+                   clang::tooling::TranslationUnitReplacements &TU);
+void loadGDCFromYaml(const clang::tooling::UnifiedPath &Input,
+                     clang::dpct::GitDiffChanges &GDC);
 int save2Yaml(
     const std::vector<clang::tooling::Replacement> &Replaces,
     const std::map<clang::tooling::UnifiedPath,
@@ -39,6 +41,7 @@ void mergeAndUniqueReps(
     clang::tooling::Replacements &Replaces,
     const std::vector<clang::tooling::Replacement> &PreRepls);
 
+bool tryLoadingUpstreamChangesAndUserChanges();
 } // namespace dpct
 } // namespace clang