From 03e98d4a3de79c769105d177c222103171aec4e9 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 19 Feb 2025 13:28:05 +0000 Subject: [PATCH 01/22] [DTLTO][LLVM] Store the Target Triple for ThinLTO modules. This will be used for supplying a `-target=` option to clang for DTLTO. Note that If DTLTO is changed to use an optimisation tool that does not require an explicit triple to be passed then the triple handling can be removed entirely. --- llvm/include/llvm/LTO/LTO.h | 7 +++++-- llvm/lib/LTO/LTO.cpp | 14 +++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 242a05f7d32c0..677ec412afc19 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -228,7 +228,8 @@ class ThinBackendProc { const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map &ResolvedODR, - MapVector &ModuleMap) = 0; + MapVector &ModuleMap, + DenseMap &ModuleTriples) = 0; Error wait() { BackendThreadPool.wait(); if (Err) @@ -426,6 +427,7 @@ class LTO { // The bitcode modules to compile, if specified by the LTO Config. std::optional ModulesToCompile; DenseMap PrevailingModuleForGUID; + DenseMap ModuleTriples; } ThinLTO; // The global resolution for a particular (mangled) symbol name. This is in @@ -517,7 +519,8 @@ class LTO { bool LivenessFromIndex); Error addThinLTO(BitcodeModule BM, ArrayRef Syms, - const SymbolResolution *&ResI, const SymbolResolution *ResE); + const SymbolResolution *&ResI, const SymbolResolution *ResE, + StringRef Triple); Error runRegularLTO(AddStreamFn AddStream); Error runThinLTO(AddStreamFn AddStream, FileCache Cache, diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 0f53c60851217..e5bc79297ecaa 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -783,7 +783,7 @@ Error LTO::addModule(InputFile &Input, unsigned ModI, LTOInfo->HasSummary); if (IsThinLTO) - return addThinLTO(BM, ModSyms, ResI, ResE); + return addThinLTO(BM, ModSyms, ResI, ResE, Input.getTargetTriple()); RegularLTO.EmptyCombinedModule = false; Expected ModOrErr = @@ -1030,7 +1030,7 @@ Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod, // Add a ThinLTO module to the link. Error LTO::addThinLTO(BitcodeModule BM, ArrayRef Syms, const SymbolResolution *&ResI, - const SymbolResolution *ResE) { + const SymbolResolution *ResE, StringRef Triple) { const SymbolResolution *ResITmp = ResI; for (const InputFile::Symbol &Sym : Syms) { assert(ResITmp != ResE); @@ -1090,6 +1090,8 @@ Error LTO::addThinLTO(BitcodeModule BM, ArrayRef Syms, "Expected at most one ThinLTO module per bitcode file", inconvertibleErrorCode()); + ThinLTO.ModuleTriples.insert({BM.getModuleIdentifier(), Triple.str()}); + if (!Conf.ThinLTOModulesToCompile.empty()) { if (!ThinLTO.ModulesToCompile) ThinLTO.ModulesToCompile = ModuleMapType(); @@ -1496,7 +1498,8 @@ class InProcessThinBackend : public ThinBackendProc { const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map &ResolvedODR, - MapVector &ModuleMap) override { + MapVector &ModuleMap, + DenseMap & /*ModuleTriples*/) override { StringRef ModulePath = BM.getModuleIdentifier(); assert(ModuleToDefinedGVSummaries.count(ModulePath)); const GVSummaryMapTy &DefinedGlobals = @@ -1776,7 +1779,8 @@ class WriteIndexesThinBackend : public ThinBackendProc { const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map &ResolvedODR, - MapVector &ModuleMap) override { + MapVector &ModuleMap, + DenseMap & /*ModuleTriples*/) override { StringRef ModulePath = BM.getModuleIdentifier(); // The contents of this file may be used as input to a native link, and must @@ -2013,7 +2017,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, return BackendProcess->start( RegularLTO.ParallelCodeGenParallelismLevel + I, Mod.second, ImportLists[Mod.first], ExportLists[Mod.first], - ResolvedODR[Mod.first], ThinLTO.ModuleMap); + ResolvedODR[Mod.first], ThinLTO.ModuleMap, ThinLTO.ModuleTriples); }; if (BackendProcess->getThreadCount() == 1 || From 19ef1d1805b8b9247c4e1a09f253e98d6d008bdf Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 19 Feb 2025 13:43:22 +0000 Subject: [PATCH 02/22] [DTLTO][LLVM] Make the ThinLTO backend wait() method virtual The DTLTO ThinLTO backend will override this function to perform code generation. The DTLTO ThinLTO backend will not do any codegen when invoked for each task. Instead, it will generate the required information (e.g., the summary index shard, import list, etc.) to allow for the codegen to be performed externally. The backend's `wait` function will then invoke an external distributor process to do backend compilations. --- llvm/include/llvm/LTO/LTO.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 677ec412afc19..41f55e91c7ed9 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -230,7 +230,7 @@ class ThinBackendProc { const std::map &ResolvedODR, MapVector &ModuleMap, DenseMap &ModuleTriples) = 0; - Error wait() { + virtual Error wait() { BackendThreadPool.wait(); if (Err) return std::move(*Err); From bc7f32e19a10b64d690f599a59673dc15627160b Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 19 Feb 2025 13:56:24 +0000 Subject: [PATCH 03/22] [DTLTO][LLVM] Generalize the emit files infrastructure The DTLTO ThinLTO backend will need to customize the file paths used for the summary index shard files. This is so that the names of these files can include a unique ID so that multiple DTLTO links do not overwrite each other's summary index shard files. It also needs to be able to get the import files list for each job in memory rather than writing it to a file. This is to allow the import lists to be included in the JSON used to communicate with the external distributor process. --- llvm/include/llvm/LTO/LTO.h | 13 +++++++++-- .../llvm/Transforms/IPO/FunctionImport.h | 6 +++++ llvm/lib/LTO/LTO.cpp | 22 ++++++++++++++++--- llvm/lib/Transforms/IPO/FunctionImport.cpp | 14 ++++++++++-- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 41f55e91c7ed9..31d939d514491 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -199,6 +199,8 @@ class InputFile { using IndexWriteCallback = std::function; +using ImportsFilesContainer = llvm::SmallVector; + /// This class defines the interface to the ThinLTO backend. class ThinBackendProc { protected: @@ -241,8 +243,15 @@ class ThinBackendProc { // Write sharded indices and (optionally) imports to disk Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, - llvm::StringRef ModulePath, - const std::string &NewModulePath) const; + StringRef ModulePath, const std::string &NewModulePath) const; + + // Write sharded indices to SummaryPath, (optionally) imports + // IndexPath, and (optionally) record imports in ImportsFiles. + Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, + StringRef ModulePath, StringRef SummaryPath, + const std::string &NewModulePath, + std::optional> + ImportsFiles) const; }; /// This callable defines the behavior of a ThinLTO backend after the thin-link diff --git a/llvm/include/llvm/Transforms/IPO/FunctionImport.h b/llvm/include/llvm/Transforms/IPO/FunctionImport.h index 3623f9194d4d1..5e4116834b7f2 100644 --- a/llvm/include/llvm/Transforms/IPO/FunctionImport.h +++ b/llvm/include/llvm/Transforms/IPO/FunctionImport.h @@ -421,6 +421,12 @@ Error EmitImportsFiles( StringRef ModulePath, StringRef OutputFilename, const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex); +/// Call \p F passing each of the files module \p ModulePath will import from. +void processImportsFiles( + StringRef ModulePath, + const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex, + function_ref F); + /// Based on the information recorded in the summaries during global /// summary-based analysis: /// 1. Resolve prevailing symbol linkages and constrain visibility (CanAutoHide diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index e5bc79297ecaa..3c1e7f35606b2 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1392,6 +1392,16 @@ SmallVector LTO::getRuntimeLibcallSymbols(const Triple &TT) { Error ThinBackendProc::emitFiles( const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath, const std::string &NewModulePath) const { + return emitFiles(ImportList, ModulePath, NewModulePath + ".thinlto.bc", + NewModulePath, + /*ImportsFiles=*/std::nullopt); +} + +Error ThinBackendProc::emitFiles( + const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath, + StringRef SummaryPath, const std::string &NewModulePath, + std::optional> ImportsFiles) + const { ModuleToSummariesForIndexTy ModuleToSummariesForIndex; GVSummaryPtrSet DeclarationSummaries; @@ -1400,10 +1410,9 @@ Error ThinBackendProc::emitFiles( ImportList, ModuleToSummariesForIndex, DeclarationSummaries); - raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC, - sys::fs::OpenFlags::OF_None); + raw_fd_ostream OS(SummaryPath, EC, sys::fs::OpenFlags::OF_None); if (EC) - return createFileError("cannot open " + NewModulePath + ".thinlto.bc", EC); + return createFileError("cannot open " + Twine(SummaryPath), EC); writeIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex, &DeclarationSummaries); @@ -1414,6 +1423,13 @@ Error ThinBackendProc::emitFiles( if (ImportFilesError) return ImportFilesError; } + + // Optionally, store the imports files. + if (ImportsFiles) + processImportsFiles( + ModulePath, ModuleToSummariesForIndex, + [&](StringRef M) { ImportsFiles->get().push_back(M.str()); }); + return Error::success(); } diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp index c3d0a1a3a046e..cdcf918d3fae8 100644 --- a/llvm/lib/Transforms/IPO/FunctionImport.cpp +++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp @@ -1568,13 +1568,23 @@ Error llvm::EmitImportsFiles( if (EC) return createFileError("cannot open " + OutputFilename, errorCodeToError(EC)); + processImportsFiles(ModulePath, ModuleToSummariesForIndex, + [&](StringRef M) { ImportsOS << M << "\n"; }); + return Error::success(); +} + +/// Invoke callback \p F on the file paths from which \p ModulePath +/// will import. +void llvm::processImportsFiles( + StringRef ModulePath, + const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex, + function_ref F) { for (const auto &ILI : ModuleToSummariesForIndex) // The ModuleToSummariesForIndex map includes an entry for the current // Module (needed for writing out the index files). We don't want to // include it in the imports file, however, so filter it out. if (ILI.first != ModulePath) - ImportsOS << ILI.first << "\n"; - return Error::success(); + F(ILI.first); } bool llvm::convertToDeclaration(GlobalValue &GV) { From 7d0c1c84b2728fc11f984123de70fdcd552a464a Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 19 Feb 2025 14:04:57 +0000 Subject: [PATCH 04/22] [DTLTO][LLVM] Add a setup() member to the ThinLTO backends The new setup() member is now called to allow the ThinLTO backends to prepare for code generation. For the DTLTO backend, this will be used to preallocate storage for the information required to perform the backend compilation jobs. --- llvm/include/llvm/LTO/LTO.h | 1 + llvm/lib/LTO/LTO.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 31d939d514491..815abecbdcda5 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -225,6 +225,7 @@ class ThinBackendProc { BackendThreadPool(ThinLTOParallelism) {} virtual ~ThinBackendProc() = default; + virtual void setup(unsigned MaxTasks) {} virtual Error start( unsigned Task, BitcodeModule BM, const FunctionImporter::ImportMapTy &ImportList, diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 3c1e7f35606b2..ad6712baa0dd6 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -2036,6 +2036,8 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, ResolvedODR[Mod.first], ThinLTO.ModuleMap, ThinLTO.ModuleTriples); }; + BackendProcess->setup(ModuleMap.size()); + if (BackendProcess->getThreadCount() == 1 || BackendProcess->isSensitiveToInputOrder()) { // Process the modules in the order they were provided on the From d55d8c02add633af1ce9de639222be2878c2c800 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 19 Feb 2025 14:14:57 +0000 Subject: [PATCH 05/22] [DTLTO][LLVM] Derive the InProcess backend from a base class Move some setup code and state to a base class. This will allow the DTLTO ThinLTO backend to share this setup code and state by also deriving from this base class. --- llvm/lib/LTO/LTO.cpp | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index ad6712baa0dd6..cb4f87358c19c 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1434,25 +1434,23 @@ Error ThinBackendProc::emitFiles( } namespace { -class InProcessThinBackend : public ThinBackendProc { +class CGThinBackend : public ThinBackendProc { protected: AddStreamFn AddStream; - FileCache Cache; DenseSet CfiFunctionDefs; DenseSet CfiFunctionDecls; - bool ShouldEmitIndexFiles; public: - InProcessThinBackend( + CGThinBackend( const Config &Conf, ModuleSummaryIndex &CombinedIndex, - ThreadPoolStrategy ThinLTOParallelism, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite, - bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles) + AddStreamFn AddStream, lto::IndexWriteCallback OnWrite, + bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, + ThreadPoolStrategy ThinLTOParallelism) : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism), - AddStream(std::move(AddStream)), Cache(std::move(Cache)), + AddStream(std::move(AddStream)), ShouldEmitIndexFiles(ShouldEmitIndexFiles) { for (auto &Name : CombinedIndex.cfiFunctionDefs()) CfiFunctionDefs.insert( @@ -1461,6 +1459,23 @@ class InProcessThinBackend : public ThinBackendProc { CfiFunctionDecls.insert( GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); } +}; + +class InProcessThinBackend : public CGThinBackend { +protected: + FileCache Cache; + +public: + InProcessThinBackend( + const Config &Conf, ModuleSummaryIndex &CombinedIndex, + ThreadPoolStrategy ThinLTOParallelism, + const DenseMap &ModuleToDefinedGVSummaries, + AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite, + bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles) + : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, + AddStream, OnWrite, ShouldEmitIndexFiles, + ShouldEmitImportsFiles, ThinLTOParallelism), + Cache(std::move(Cache)) {} virtual Error runThinLTOBackendThread( AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM, From e3c016fe0ac3b3eb43b0f12096770fa42b304d7a Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Mon, 17 Feb 2025 17:59:02 +0000 Subject: [PATCH 06/22] [DTLTO][LLVM] Implement integrated distribution for ThinLTO (DTLTO). Structural changes: 1. A new ThinLTO backend implementing DTLTO has been added. 2. Both the new backend and the InProcess backend derive from a common base class to share common setup code and state. 3. The target triple is now stored for the ThinLTO bitcode files. 4. A new setup() member is called, which the ThinLTO backends can use to prepare for code generation. For the DTLTO backend, this is used to pre-allocate storage for the information required to perform the backend compilation jobs. 5. The functions for emitting summary index shard and imports files have been altered to allow the caller to specify the filenames to write and to allow the list of imports to be stored in a container rather than written to a file. --- llvm/include/llvm/LTO/LTO.h | 22 ++ llvm/lib/LTO/LTO.cpp | 277 +++++++++++++++++++++++ llvm/test/ThinLTO/X86/dtlto/dtlto.test | 85 +++++++ llvm/test/ThinLTO/X86/dtlto/imports.test | 79 +++++++ llvm/test/ThinLTO/X86/dtlto/json.test | 99 ++++++++ llvm/test/ThinLTO/X86/dtlto/summary.test | 55 +++++ llvm/test/ThinLTO/X86/dtlto/triple.test | 44 ++++ llvm/test/lit.cfg.py | 1 + llvm/tools/llvm-lto2/llvm-lto2.cpp | 22 +- llvm/utils/dtlto/mock.py | 20 ++ llvm/utils/dtlto/validate.py | 79 +++++++ 11 files changed, 782 insertions(+), 1 deletion(-) create mode 100644 llvm/test/ThinLTO/X86/dtlto/dtlto.test create mode 100644 llvm/test/ThinLTO/X86/dtlto/imports.test create mode 100644 llvm/test/ThinLTO/X86/dtlto/json.test create mode 100644 llvm/test/ThinLTO/X86/dtlto/summary.test create mode 100644 llvm/test/ThinLTO/X86/dtlto/triple.test create mode 100644 llvm/utils/dtlto/mock.py create mode 100644 llvm/utils/dtlto/validate.py diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 815abecbdcda5..c776b57c1af82 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -305,6 +305,28 @@ ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, bool ShouldEmitIndexFiles = false, bool ShouldEmitImportsFiles = false); +/// This ThinBackend generates the index shards and then runs the individual +/// backend jobs via an external process. It takes the same parameters as the +/// InProcessThinBackend, however, these parameters only control the behavior +/// when generating the index files for the modules. Addtionally: +/// LinkerOutputFile is a string that should identify this LTO invocation in +/// the context of a wider build. It's used for naming to aid the user in +/// identifying activity related to a specific LTO invocation. +/// RemoteOptTool specifies the path to a Clang executable to be invoked for the +/// backend jobs. +/// Distributor specifies the path to a process to invoke to manage the backend +/// jobs execution. +/// SaveTemps is a debugging tool that prevents temporary files created by this +/// backend from being cleaned up. +ThinBackend createOutOfProcessThinBackend(ThreadPoolStrategy Parallelism, + IndexWriteCallback OnWrite, + bool ShouldEmitIndexFiles, + bool ShouldEmitImportsFiles, + StringRef LinkerOutputFile, + StringRef RemoteOptTool, + StringRef Distributor, + bool SaveTemps); + /// This ThinBackend writes individual module indexes to files, instead of /// running the individual backend jobs. This backend is for distributed builds /// where separate processes will invoke the real backends. diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index cb4f87358c19c..3a9d57f69f718 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -41,8 +41,11 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/FileUtilities.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" #include "llvm/Support/SHA1.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/ThreadPool.h" @@ -91,6 +94,15 @@ extern cl::opt SupportsHotColdNew; /// Enable MemProf context disambiguation for thin link. extern cl::opt EnableMemProfContextDisambiguation; + +cl::list AdditionalThinLTODistributorArgs( + "thinlto-distributor-arg", + cl::desc("Additional arguments to pass to the ThinLTO distributor")); + +cl::list + ThinLTORemoteOptToolArgs("thinlto-remote-opt-tool-arg", + cl::desc("Additional arguments to pass to the " + "ThinLTO remote optimization tool")); } // namespace llvm // Computes a unique hash for the Module considering the current list of @@ -2179,3 +2191,268 @@ std::vector lto::generateModulesOrdering(ArrayRef R) { }); return ModulesOrdering; } + +namespace { +// For this out-of-process backend no codegen is done when invoked for each +// task. Instead we generate the required information (e.g. the summary index +// shard, import list, etc..) to allow for the codegen to be performed +// externally . This backend's `wait` function then invokes an external +// distributor process to do backend compilations. +class OutOfProcessThinBackend : public CGThinBackend { + using SString = SmallString<128>; + + BumpPtrAllocator Alloc; + StringSaver Saver{Alloc}; + + SString LinkerOutputFile; + SString RemoteOptTool; + SString DistributorPath; + bool SaveTemps; + + SmallVector CodegenOptions; + DenseSet AdditionalInputs; + + // Information specific to individual backend compilation job. + struct Job { + unsigned Task; + StringRef ModuleID; + StringRef Triple; + StringRef NativeObjectPath; + StringRef SummaryIndexPath; + ImportsFilesContainer ImportFiles; + }; + // The set of backend compilations jobs. + SmallVector Jobs; + + // A unique string to identify the current link. + SmallString<8> UID; + +public: + OutOfProcessThinBackend( + const Config &Conf, ModuleSummaryIndex &CombinedIndex, + ThreadPoolStrategy ThinLTOParallelism, + const DenseMap &ModuleToDefinedGVSummaries, + AddStreamFn AddStream, lto::IndexWriteCallback OnWrite, + bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, + StringRef LinkerOutputFile, StringRef RemoteOptTool, + StringRef Distributor, bool SaveTemps) + : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, + AddStream, OnWrite, ShouldEmitIndexFiles, + ShouldEmitImportsFiles, ThinLTOParallelism), + LinkerOutputFile(LinkerOutputFile), RemoteOptTool(RemoteOptTool), + DistributorPath(Distributor), SaveTemps(SaveTemps) {} + + virtual void setup(unsigned MaxTasks) override { + UID = itostr(sys::Process::getProcessId()); + Jobs.resize((size_t)MaxTasks); + } + + Error start( + unsigned Task, BitcodeModule BM, + const FunctionImporter::ImportMapTy &ImportList, + const FunctionImporter::ExportSetTy &ExportList, + const std::map &ResolvedODR, + MapVector &ModuleMap, + DenseMap &ModuleTriples) override { + + StringRef ModulePath = BM.getModuleIdentifier(); + + SString ObjFilePath = sys::path::parent_path(LinkerOutputFile); + sys::path::append(ObjFilePath, sys::path::stem(ModulePath) + "." + + itostr(Task) + "." + UID + ".native.o"); + + Job &J = Jobs[Task - 1]; /*Task 0 is reserved*/ + J = {Task, + ModulePath, + ModuleTriples[ModulePath], + Saver.save(ObjFilePath.str()), + Saver.save(ObjFilePath.str() + ".thinlto.bc"), + {}}; + + assert(ModuleToDefinedGVSummaries.count(ModulePath)); + BackendThreadPool.async( + [=](Job &J, const FunctionImporter::ImportMapTy &ImportList) { + if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled) + timeTraceProfilerInitialize(Conf.TimeTraceGranularity, + "thin backend"); + if (auto E = emitFiles(ImportList, J.ModuleID, J.SummaryIndexPath, + J.ModuleID.str(), J.ImportFiles)) { + std::unique_lock L(ErrMu); + if (Err) + Err = joinErrors(std::move(*Err), std::move(E)); + else + Err = std::move(E); + } + if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled) + timeTraceProfilerFinishThread(); + }, + std::ref(J), std::ref(ImportList)); + + return Error::success(); + } + + // Generates a JSON file describing the backend compilations, for the + // distributor. + bool emitDistributorJson(StringRef DistributorJson) { + using json::Array; + std::error_code EC; + raw_fd_ostream OS(DistributorJson, EC); + if (EC) + return false; + + json::OStream JOS(OS); + JOS.object([&]() { + // Information common to all jobs note that we use a custom syntax for + // referencing by index into the job input and output file arrays. + JOS.attributeObject("common", [&]() { + JOS.attribute("linker_output", LinkerOutputFile); + + // Common command line template. + JOS.attributeArray("args", [&]() { + JOS.value(RemoteOptTool); + + // Reference to Job::NativeObjectPath. + JOS.value("-o"); + JOS.value(Array{"primary_output", 0}); + + JOS.value("-c"); + + JOS.value("-x"); + JOS.value("ir"); + + // Reference to Job::ModuleID. + JOS.value(Array{"primary_input", 0}); + + // Reference to Job::SummaryIndexPath. + JOS.value(Array{"summary_index", "-fthinlto-index=", 0}); + JOS.value(Saver.save("--target=" + Twine(Jobs.front().Triple))); + + for (const auto &A : ThinLTORemoteOptToolArgs) + JOS.value(A); + }); + }); + JOS.attributeArray("jobs", [&]() { + for (const auto &J : Jobs) { + assert(J.Task != 0); + JOS.object([&]() { + JOS.attribute("primary_input", Array{J.ModuleID}); + JOS.attribute("summary_index", Array{J.SummaryIndexPath}); + JOS.attribute("primary_output", Array{J.NativeObjectPath}); + + // Add the bitcode files from which imports will be made. These do + // not appear on the command line but are recorded in the summary + // index shard. + JOS.attribute("imports", Array(J.ImportFiles)); + + // Add any input files that are common to each invocation. These + // filenames are duplicated in the command line template and in + // each of the per job "inputs" array. However, this small amount + // of duplication makes the schema simpler. + JOS.attribute("additional_inputs", Array(AdditionalInputs)); + }); + } + }); + }); + + return true; + } + + void removeFile(StringRef FileName) { + std::error_code EC = sys::fs::remove(FileName, true); + if (EC && EC != std::make_error_code(std::errc::no_such_file_or_directory)) + errs() << "warning: could not remove the file '" << FileName + << "': " << EC.message() << "\n"; + } + + Error wait() override { + // Wait for the information on the required backend compilations to be + // gathered. + BackendThreadPool.wait(); + if (Err) + return std::move(*Err); + + auto CleanPerJobFiles = llvm::make_scope_exit([&] { + if (!SaveTemps) + for (auto &Job : Jobs) { + removeFile(Job.NativeObjectPath); + if (!ShouldEmitIndexFiles) + removeFile(Job.SummaryIndexPath); + } + }); + + const StringRef BCError = "DTLTO backend compilation: "; + + // TODO: If we move to using an optimisation tool that does not require an + // explicit triple to be passed then the triple handling can be removed + // entirely. + if (!llvm::all_of(Jobs, [&](const auto &Job) { + return Job.Triple == Jobs.front().Triple; + })) + return make_error(BCError + "all triples must be consistent", + inconvertibleErrorCode()); + + SString JsonFile = sys::path::parent_path(LinkerOutputFile); + sys::path::append(JsonFile, sys::path::stem(LinkerOutputFile) + "." + UID + + ".dist-file.json"); + if (!emitDistributorJson(JsonFile)) + return make_error( + BCError + "failed to generate distributor JSON script: " + JsonFile, + inconvertibleErrorCode()); + auto CleanJson = llvm::make_scope_exit([&] { + if (!SaveTemps) + removeFile(JsonFile); + }); + + SmallVector Args = {DistributorPath}; + llvm::append_range(Args, AdditionalThinLTODistributorArgs); + Args.push_back(JsonFile); + std::string ErrMsg; + if (sys::ExecuteAndWait(Args[0], Args, + /*Env=*/std::nullopt, /*Redirects=*/{}, + /*SecondsToWait=*/0, /*MemoryLimit=*/0, &ErrMsg)) { + return make_error( + BCError + "distributor execution failed" + + (!ErrMsg.empty() ? ": " + ErrMsg + Twine(".") : Twine(".")), + inconvertibleErrorCode()); + } + + for (auto &Job : Jobs) { + // Load the native object from a file into a memory buffer + // and store its contents in the output buffer. + ErrorOr> objFileMbOrErr = + MemoryBuffer::getFile(Job.NativeObjectPath, false, false); + if (std::error_code ec = objFileMbOrErr.getError()) + return make_error( + BCError + "cannot open native object file: " + + Job.NativeObjectPath + ": " + ec.message(), + inconvertibleErrorCode()); + std::unique_ptr umb = std::move(objFileMbOrErr.get()); + Expected> StreamOrErr = + AddStream(Job.Task, Job.ModuleID); + if (Error Err = StreamOrErr.takeError()) + report_fatal_error(std::move(Err)); + std::unique_ptr Stream = std::move(*StreamOrErr); + *Stream->OS << umb->getMemBufferRef().getBuffer(); + } + + return Error::success(); + } +}; +} // end anonymous namespace + +ThinBackend lto::createOutOfProcessThinBackend( + ThreadPoolStrategy Parallelism, lto::IndexWriteCallback OnWrite, + bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, + StringRef LinkerOutputFile, StringRef RemoteOptTool, StringRef Distributor, + bool SaveTemps) { + auto Func = + [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, + const DenseMap &ModuleToDefinedGVSummaries, + AddStreamFn AddStream, FileCache /*Cache*/) { + return std::make_unique( + Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, + AddStream, OnWrite, ShouldEmitIndexFiles, ShouldEmitImportsFiles, + LinkerOutputFile, RemoteOptTool, Distributor, SaveTemps); + }; + return ThinBackend(Func, Parallelism); +} diff --git a/llvm/test/ThinLTO/X86/dtlto/dtlto.test b/llvm/test/ThinLTO/X86/dtlto/dtlto.test new file mode 100644 index 0000000000000..17dde95c022d8 --- /dev/null +++ b/llvm/test/ThinLTO/X86/dtlto/dtlto.test @@ -0,0 +1,85 @@ +# Test DTLTO output with llvm-lto2. + +RUN: rm -rf %t && split-file %s %t && cd %t + +# Generate bitcode files with summary. +RUN: opt -thinlto-bc t1.ll -o t1.bc +RUN: opt -thinlto-bc t2.ll -o t2.bc + +# Generate mock native object files. +RUN: opt t1.ll -o t1.o +RUN: opt t2.ll -o t2.o + +# Create an empty subdirectory to avoid having to account for the input files. +RUN: mkdir %t/out && cd %t/out + +# Define a substitution to share the common DTLTO arguments. +DEFINE: %{command} =llvm-lto2 run ../t1.bc ../t2.bc -o t.o \ +DEFINE: -dtlto \ +DEFINE: -dtlto-remote-opt-tool=dummy \ +DEFINE: -dtlto-distributor=%python \ +DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +DEFINE: -thinlto-distributor-arg=../t1.o \ +DEFINE: -thinlto-distributor-arg=../t2.o \ +DEFINE: -r=../t1.bc,t1,px \ +DEFINE: -r=../t2.bc,t2,px + +# Perform DTLTO. mock.py does not do any compilation, instead it simply writes +# the contents of the object files supplied on the command line into the +# output object files in job order. +RUN: %{command} + +# Check that the expected output files have been created. +RUN: ls | count 2 +RUN: ls | FileCheck %s --check-prefix=THINLTO + +# llvm-lto2 ThinLTO output files. +THINLTO-DAG: {{^}}t.o.1{{$}} +THINLTO-DAG: {{^}}t.o.2{{$}} + +RUN: cd .. && rm -rf %t/out && mkdir %t/out && cd %t/out + +# Perform DTLTO with --save-temps. +RUN: %{command} --save-temps + +# Check that the expected output files have been created. +RUN: ls | count 12 +RUN: ls | FileCheck %s --check-prefixes=THINLTO,SAVETEMPS + +# Common -save-temps files from llvm-lto2. +SAVETEMPS-DAG: {{^}}t.o.resolution.txt{{$}} +SAVETEMPS-DAG: {{^}}t.o.index.bc{{$}} +SAVETEMPS-DAG: {{^}}t.o.index.dot{{$}} + +# -save-temps incremental files. +SAVETEMPS-DAG: {{^}}t.o.0.0.preopt.bc{{$}} +SAVETEMPS-DAG: {{^}}t.o.0.2.internalize.bc{{$}} + +# A jobs description JSON. +SAVETEMPS-DAG: {{^}}t.[[#]].dist-file.json{{$}} + +# Summary shards emitted for DTLTO. +SAVETEMPS-DAG: {{^}}t1.1.[[#]].native.o.thinlto.bc{{$}} +SAVETEMPS-DAG: {{^}}t2.2.[[#]].native.o.thinlto.bc{{$}} + +# DTLTO native output files (the results of the external backend compilations). +SAVETEMPS-DAG: {{^}}t1.1.[[#]].native.o{{$}} +SAVETEMPS-DAG: {{^}}t2.2.[[#]].native.o{{$}} + +#--- t1.ll + +target triple = "x86_64-unknown-linux-gnu" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define void @t1() { + ret void +} + +#--- t2.ll + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t2() { + ret void +} diff --git a/llvm/test/ThinLTO/X86/dtlto/imports.test b/llvm/test/ThinLTO/X86/dtlto/imports.test new file mode 100644 index 0000000000000..57ace00aeaf9e --- /dev/null +++ b/llvm/test/ThinLTO/X86/dtlto/imports.test @@ -0,0 +1,79 @@ +# Check that DTLTO creates imports lists correctly. + +RUN: rm -rf %t && split-file %s %t && cd %t + +# Compile bitcode. +RUN: opt -thinlto-bc 0.ll -o 0.bc -O2 +RUN: opt -thinlto-bc 1.ll -o 1.bc -O2 + +# Define a substitution to share the common DTLTO arguments. Note that the use +# of validate.py will cause a failure as it does not create output files. +DEFINE: %{command} =llvm-lto2 run \ +DEFINE: 0.bc 1.bc -o t.o \ +DEFINE: -dtlto \ +DEFINE: -dtlto-remote-opt-tool=dummy \ +DEFINE: -dtlto-distributor=%python \ +DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \ +DEFINE: -thinlto-distributor-arg=0.bc \ +DEFINE: -thinlto-distributor-arg=1.bc \ +DEFINE: -thinlto-emit-indexes \ +DEFINE: -r=0.bc,g,px \ +DEFINE: -r=1.bc,f,px \ +DEFINE: -r=1.bc,g + +# We expect an import from 0.o into 1.o but no imports into 0.o. Check that the +# expected input files have been added to the JSON. +RUN: not %{command} >out.log 2>&1 +RUN: FileCheck --input-file=out.log %s --check-prefixes=INPUTS,ERR + +INPUTS: "primary_input": [ +INPUTS-NEXT: "0.bc" +INPUTS-NEXT: ] +INPUTS: "imports": [] +INPUTS: "primary_input": [ +INPUTS-NEXT: "1.bc" +INPUTS-NEXT: ] +INPUTS: "imports": [ +INPUTS-NEXT: "0.bc" +INPUTS-NEXT: ] + +# This check ensures that we have failed for the expected reason. +ERR: failed: DTLTO backend compilation: cannot open native object file: + +# Check that imports files have not been created. +RUN: ls | FileCheck %s --check-prefix=NOINDEXFILES +NOINDEXFILES-NOT: imports + +# Check that imports files are not created with -save-temps. +RUN: not %{command} -save-temps 2>&1 \ +RUN: | FileCheck %s --check-prefixes=ERR +RUN: ls | FileCheck %s --check-prefix=NOINDEXFILES +NOINDEXFILES-NOT: imports + +# Check that imports files are created with -thinlto-emit-imports. +RUN: not %{command} -thinlto-emit-imports 2>&1 \ +RUN: | FileCheck %s --check-prefixes=ERR +RUN: ls | FileCheck %s --check-prefix=INDEXFILES +INDEXFILES: 0.bc.imports +INDEXFILES: 1.bc.imports + +#--- 0.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @g() { +entry: + ret void +} + +#--- 1.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +declare void @g(...) + +define void @f() { +entry: + call void (...) @g() + ret void +} diff --git a/llvm/test/ThinLTO/X86/dtlto/json.test b/llvm/test/ThinLTO/X86/dtlto/json.test new file mode 100644 index 0000000000000..f6a78957cda33 --- /dev/null +++ b/llvm/test/ThinLTO/X86/dtlto/json.test @@ -0,0 +1,99 @@ +# Check that the JSON output from DTLTO is as expected. Note that validate.py +# checks the JSON structure so we just check the field contents in this test. + +RUN: rm -rf %t && split-file %s %t && cd %t + +# Generate bitcode files with summary. +RUN: opt -thinlto-bc t1.ll -o t1.bc +RUN: opt -thinlto-bc t2.ll -o t2.bc + +# Perform DTLTO. +RUN: not llvm-lto2 run t1.bc t2.bc -o my.output \ +RUN: -r=t1.bc,t1,px -r=t2.bc,t2,px \ +RUN: -dtlto \ +RUN: -dtlto-remote-opt-tool=my_clang.exe \ +RUN: -dtlto-distributor=%python \ +RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \ +RUN: -thinlto-remote-opt-tool-arg=--rota1=10 \ +RUN: -thinlto-remote-opt-tool-arg=--rota2=20 \ +RUN: -thinlto-distributor-arg=--da1=10 \ +RUN: -thinlto-distributor-arg=--da2=10 \ +RUN: 2>&1 | FileCheck %s + +CHECK: distributor_args=['--da1=10', '--da2=10'] + +# Check the common object. +CHECK: "linker_output": "my.output" +CHECK: "args": +CHECK: "my_clang.exe" +CHECK: "-o" +CHECK-NEXT: [ +CHECK-NEXT: "primary_output" +CHECK-NEXT: 0 +CHECK-NEXT: ] +CHECK: "-c" +CHECK: "-x" +CHECK: "ir" +CHECK-NEXT: [ +CHECK-NEXT: "primary_input" +CHECK-NEXT: 0 +CHECK-NEXT: ] +CHECK: "summary_index" +CHECK-NEXT: "-fthinlto-index=" +CHECK-NEXT: 0 +CHECK-NEXT: ] +CHECK: "--target=x86_64-unknown-linux-gnu" +CHECK: "--rota1=10" +CHECK: "--rota2=20" + +# Check the first job entry. +CHECK: "jobs": +CHECK: "primary_input": [ +CHECK-NEXT: "t1.bc" +CHECK-NEXT: ] +CHECK: "summary_index": [ +CHECK-NEXT: "t1.1.[[#]].native.o.thinlto.bc" +CHECK-NEXT: ] +CHECK: "primary_output": [ +CHECK-NEXT: "t1.1.[[#]].native.o" +CHECK-NEXT: ] +CHECK: "imports": [], +CHECK: "additional_inputs": [] +CHECK-NEXT: } + +# Check the second job entry. +CHECK-NEXT: { +CHECK-NEXT: "primary_input": [ +CHECK-NEXT: "t2.bc" +CHECK-NEXT: ] +CHECK-NEXT: "summary_index": [ +CHECK-NEXT: "t2.2.[[#]].native.o.thinlto.bc" +CHECK-NEXT: ] +CHECK-NEXT: "primary_output": [ +CHECK-NEXT: "t2.2.[[#]].native.o" +CHECK-NEXT: ] +CHECK-NEXT: "imports": [] +CHECK-NEXT: "additional_inputs": [] +CHECK-NEXT: } +CHECK-NEXT: ] + +# This check ensures that we have failed for the expected reason. +CHECK: failed: DTLTO backend compilation: cannot open native object file: + +#--- t1.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t1() { +entry: + ret void +} + +#--- t2.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t2() { +entry: + ret void +} diff --git a/llvm/test/ThinLTO/X86/dtlto/summary.test b/llvm/test/ThinLTO/X86/dtlto/summary.test new file mode 100644 index 0000000000000..dab59ece12f8e --- /dev/null +++ b/llvm/test/ThinLTO/X86/dtlto/summary.test @@ -0,0 +1,55 @@ +# Check that DTLTO creates identical summary index shard files as are created +# for an equivalent ThinLTO link. + + RUN: rm -rf %t && split-file %s %t && cd %t + +# Generate ThinLTO bitcode files. +RUN: opt -thinlto-bc t1.ll -o t1.bc +RUN: opt -thinlto-bc t2.ll -o t2.bc + +# Generate mock native object files. +RUN: opt t1.ll -o t1.o +RUN: opt t2.ll -o t2.o + +# Define a substitution to share the common arguments. +DEFINE: %{command} =llvm-lto2 run t1.bc t2.bc -o t.o \ +DEFINE: -r=t1.bc,t1,px \ +DEFINE: -r=t2.bc,t2,px \ +DEFINE: -r=t2.bc,t1 \ +DEFINE: -thinlto-emit-indexes + +# Perform DTLTO. +RUN: %{command} -dtlto \ +RUN: -dtlto-remote-opt-tool=dummy \ +RUN: -dtlto-distributor=%python \ +RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: -thinlto-distributor-arg=t1.o \ +RUN: -thinlto-distributor-arg=t2.o + +# Perform ThinLTO. +RUN: %{command} + +# Check for equivalence. We use a wildcard to account for the PID. +RUN: cmp t1.1.*.native.o.thinlto.bc t1.bc.thinlto.bc +RUN: cmp t2.2.*.native.o.thinlto.bc t2.bc.thinlto.bc + +#--- t1.ll +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @t1() { +entry: + ret void +} + +#--- t2.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +declare void @t1(...) + +define void @t2() { +entry: + call void (...) @t1() + ret void +} diff --git a/llvm/test/ThinLTO/X86/dtlto/triple.test b/llvm/test/ThinLTO/X86/dtlto/triple.test new file mode 100644 index 0000000000000..f0642281535a9 --- /dev/null +++ b/llvm/test/ThinLTO/X86/dtlto/triple.test @@ -0,0 +1,44 @@ +# Test the DTLTO limitation that all triples must match. + +RUN: rm -rf %t && split-file %s %t && cd %t + +# Generate bitcode files with summary. +RUN: opt -thinlto-bc t1.ll -o t1.bc +RUN: opt -thinlto-bc t2.ll -o t2.bc + +# Generate native object files. +RUN: opt t1.ll -o t1.o +RUN: opt t2.ll -o t2.o + +# Perform DTLTO. mock.py does not do any compilation, instead it emits the +# object files supplied using -thinlto-distributor-arg in job order. +RUN: not llvm-lto2 run t1.bc t2.bc -o t.o -save-temps \ +RUN: -dtlto \ +RUN: -dtlto-remote-opt-tool=dummy \ +RUN: -dtlto-distributor=%python \ +RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: -thinlto-distributor-arg=t1.o \ +RUN: -thinlto-distributor-arg=t2.o \ +RUN: -r=t1.bc,t1,px \ +RUN: -r=t2.bc,t2,px 2>&1 | FileCheck %s + +# This check ensures that we have failed for the expected reason. +CHECK: failed: DTLTO backend compilation: all triples must be consistent + +;--- t1.ll + +target triple = "x86_64-unknown-linux-gnu" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define void @t1() { + ret void +} + +;--- t2.ll + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-unknown-gnu" + +define void @t2() { + ret void +} diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py index aad7a088551b2..6722064d2a7b6 100644 --- a/llvm/test/lit.cfg.py +++ b/llvm/test/lit.cfg.py @@ -91,6 +91,7 @@ def get_asan_rtlib(): config.substitutions.append(("%shlibext", config.llvm_shlib_ext)) config.substitutions.append(("%pluginext", config.llvm_plugin_ext)) config.substitutions.append(("%exeext", config.llvm_exe_ext)) +config.substitutions.append(("%llvm_src_root", config.llvm_src_root)) lli_args = [] diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp index d4f022ef021a4..82f2db58acd11 100644 --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -97,6 +97,16 @@ static cl::opt "specified with -thinlto-emit-indexes or " "-thinlto-distributed-indexes")); +static cl::opt DTLTO("dtlto", cl::desc("Perform DTLTO")); + +static cl::opt + DTLTORemoteOptTool("dtlto-remote-opt-tool", + cl::desc("Specify the remote opt tool for DTLTO")); + +static cl::opt + DTLTODistributor("dtlto-distributor", + cl::desc("Specify the distributor for DTLTO")); + // Default to using all available threads in the system, but using only one // thread per core (no SMT). // Use -thinlto-threads=all to use hardware_concurrency() instead, which means @@ -344,6 +354,10 @@ static int run(int argc, char **argv) { Conf.PTO.LoopVectorization = Conf.OptLevel > 1; Conf.PTO.SLPVectorization = Conf.OptLevel > 1; + if (ThinLTODistributedIndexes && DTLTO) + llvm::errs() << "-thinlto-distributed-indexes cannot be specfied together " + "with -dtlto\n"; + ThinBackend Backend; if (ThinLTODistributedIndexes) Backend = createWriteIndexesThinBackend(llvm::hardware_concurrency(Threads), @@ -353,7 +367,13 @@ static int run(int argc, char **argv) { ThinLTOEmitImports, /*LinkedObjectsFile=*/nullptr, /*OnWrite=*/{}); - else + else if (DTLTO) { + + Backend = createOutOfProcessThinBackend( + llvm::heavyweight_hardware_concurrency(Threads), + /*OnWrite=*/{}, ThinLTOEmitIndexes, ThinLTOEmitImports, OutputFilename, + DTLTORemoteOptTool, DTLTODistributor, SaveTemps); + } else Backend = createInProcessThinBackend( llvm::heavyweight_hardware_concurrency(Threads), /* OnWrite */ {}, ThinLTOEmitIndexes, ThinLTOEmitImports); diff --git a/llvm/utils/dtlto/mock.py b/llvm/utils/dtlto/mock.py new file mode 100644 index 0000000000000..423a7f7beb194 --- /dev/null +++ b/llvm/utils/dtlto/mock.py @@ -0,0 +1,20 @@ +import sys +import json +import shutil +from pathlib import Path +import validate + +if __name__ == "__main__": + json_arg = sys.argv[-1] + distributor_args = sys.argv[1:-1] + + # Load the DTLTO information from the input JSON file. + data = json.loads(Path(json_arg).read_bytes()) + + # Iterate over the jobs and create the output + # files by copying over the supplied input files. + for job_index, job in enumerate(data["jobs"]): + shutil.copy(distributor_args[job_index], job["primary_output"][0]) + + # Check the format of the JSON. + validate.validate(data) diff --git a/llvm/utils/dtlto/validate.py b/llvm/utils/dtlto/validate.py new file mode 100644 index 0000000000000..eb14f2c746a8e --- /dev/null +++ b/llvm/utils/dtlto/validate.py @@ -0,0 +1,79 @@ +import sys +import json +from pathlib import Path + + +def take(jvalue, jpath): + parts = jpath.split(".") + for part in parts[:-1]: + jvalue = jvalue[part] + return jvalue.pop(parts[-1], KeyError) + + +def validate(jdoc): + # Check the format of the JSON + assert type(take(jdoc, "common.linker_output")) is str + + args = take(jdoc, "common.args") + assert type(args) is list + assert len(args) > 0 + + def validate_reference(a): + for j in jdoc["jobs"]: + for x in a[1:]: + if type(x) is int: + if a[0] not in j or x >= len(j[a[0]]): + return False + return True + + for a in args: + assert type(a) is str or ( + type(a) is list + and len(a) >= 2 + and type(a[0]) is str + and all(type(x) in (str, int) for x in a[1:]) + and any(type(x) is int for x in a[1:]) + and validate_reference(a) + ) + + assert len(take(jdoc, "common")) == 0 + + jobs = take(jdoc, "jobs") + assert type(jobs) is list + for j in jobs: + assert type(j) is dict + + # Mandatory job attributes. + for attr in ("primary_input", "primary_output", "summary_index"): + array = take(j, attr) + assert type(array) is list + assert len(array) == 1 + assert type(array[0]) is str + + # Optional job attributes. + for attr in ("additional_inputs", "additional_outputs", "imports"): + array = take(j, attr) + if array is KeyError: + continue + assert type(array) is list + assert all(type(a) is str for a in array) + + assert len(j) == 0 + + assert len(jdoc) == 0 + + +if __name__ == "__main__": + json_arg = sys.argv[-1] + distributor_args = sys.argv[1:-1] + + print(f"{distributor_args=}") + + # Load the DTLTO information from the input JSON file. + jdoc = json.loads(Path(json_arg).read_bytes()) + + # Write the input JSON to stdout. + print(json.dumps(jdoc, indent=4)) + + # Check the format of the JSON + validate(jdoc) From 0490b3b78f854a6c64f9ab079adb7f9440e26f77 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 19 Feb 2025 03:38:07 +0000 Subject: [PATCH 07/22] [DTLTO][LLVM] Translate some LTO configuration state into clang options. Intentionally minimal for now. Additional state translation will be added in future commits. --- cross-project-tests/CMakeLists.txt | 13 +- cross-project-tests/dtlto/README.md | 3 + .../dtlto/dtlto-translate-options.ll | 144 ++++++++++++++++++ cross-project-tests/dtlto/lit.local.cfg | 2 + cross-project-tests/lit.cfg.py | 5 +- llvm/lib/LTO/LTO.cpp | 60 +++++++- llvm/test/ThinLTO/X86/dtlto/json.test | 3 + llvm/utils/dtlto/local.py | 25 +++ 8 files changed, 251 insertions(+), 4 deletions(-) create mode 100644 cross-project-tests/dtlto/README.md create mode 100644 cross-project-tests/dtlto/dtlto-translate-options.ll create mode 100644 cross-project-tests/dtlto/lit.local.cfg create mode 100644 llvm/utils/dtlto/local.py diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt index 7f2fee48fda77..c58119447ce0e 100644 --- a/cross-project-tests/CMakeLists.txt +++ b/cross-project-tests/CMakeLists.txt @@ -19,11 +19,13 @@ set(CROSS_PROJECT_TEST_DEPS FileCheck check-gdb-llvm-support count - llvm-dwarfdump llvm-config + llvm-dwarfdump + llvm-lto2 llvm-objdump - split-file not + opt + split-file ) if ("clang" IN_LIST LLVM_ENABLE_PROJECTS) @@ -94,6 +96,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests" DEPENDS clang ) +# DTLTO tests. +add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests" + ${CMAKE_CURRENT_BINARY_DIR}/dtlto + EXCLUDE_FROM_CHECK_ALL + DEPENDS ${CROSS_PROJECT_TEST_DEPS} + ) + # Add check-cross-project-* targets. add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${CROSS_PROJECT_TEST_DEPS} diff --git a/cross-project-tests/dtlto/README.md b/cross-project-tests/dtlto/README.md new file mode 100644 index 0000000000000..cfd9d3496ca42 --- /dev/null +++ b/cross-project-tests/dtlto/README.md @@ -0,0 +1,3 @@ +Tests for DTLTO (integrated distributed ThinLTO) functionality. + +These are integration tests as DTLTO invokes `clang` for code-generation. \ No newline at end of file diff --git a/cross-project-tests/dtlto/dtlto-translate-options.ll b/cross-project-tests/dtlto/dtlto-translate-options.ll new file mode 100644 index 0000000000000..bbb6ccf33fe7c --- /dev/null +++ b/cross-project-tests/dtlto/dtlto-translate-options.ll @@ -0,0 +1,144 @@ +;; Check that the expected Clang arguments are generated by DTLTO for the +;; backend compilations and are accepted by Clang. + +; RUN: rm -rf %t && split-file %s %t && cd %t + +;; Generate bitcode files with a summary index. +; RUN: opt -thinlto-bc x86_64-unknown-linux-gnu.ll -o x86_64-unknown-linux-gnu.bc +; RUN: opt -thinlto-bc x86_64-pc-windows-msvc.ll -o x86_64-pc-windows-msvc.bc + + +;; Check that any invalid arguments would cause a Clang error. This property is +;; relied on by the actual testcases later in this test. +; RUN: not %clang -x ir x86_64-unknown-linux-gnu.ll \ +; RUN: -invalid-incorrect-not-an-option 2>&1 | FileCheck %s --check-prefix=SANITY1 +; SANITY1: unknown argument: '-invalid-incorrect-not-an-option' + + +;; Define a substitution used to simplify the testcases. +; DEFINE: %{distributor} = dummy +; DEFINE: %{extra_flags} = dummy +; DEFINE: %{triple} = dummy +; DEFINE: %{command} = llvm-lto2 run \ +; DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/%{distributor} \ +; DEFINE: -thinlto-remote-opt-tool-arg=-Wunused-command-line-argument \ +; DEFINE: @%{triple}.rsp %{extra_flags} + + +;; Write common arguments to a response files. + +; RUN: echo "x86_64-unknown-linux-gnu.bc -o x86_64-unknown-linux-gnu.o \ +; RUN: -dtlto \ +; RUN: -dtlto-remote-opt-tool=%clang \ +; RUN: -thinlto-remote-opt-tool-arg=-Werror \ +; RUN: -dtlto-distributor=%python \ +; RUN: -r=x86_64-unknown-linux-gnu.bc,globalfunc1,plx" > x86_64-unknown-linux-gnu.rsp + +; RUN: echo "x86_64-pc-windows-msvc.bc -o x86_64-pc-windows-msvc.o \ +; RUN: -dtlto \ +; RUN: -dtlto-remote-opt-tool=%clang \ +; RUN: -thinlto-remote-opt-tool-arg=-Werror \ +; RUN: -thinlto-remote-opt-tool-arg=-Wno-override-module \ +; RUN: -dtlto-distributor=%python \ +; RUN: -r=x86_64-pc-windows-msvc.bc,globalfunc2,plx" > x86_64-pc-windows-msvc.rsp + + +;; Check that boolean configuration states are translated as expected and Clang +;; accepts them. + +; RUN: echo " \ +; RUN: --addrsig=1 \ +; RUN: -function-sections=1 \ +; RUN: -data-sections=1" > on.rsp + +; RUN: echo " \ +; RUN: --addrsig=0 \ +; RUN: -function-sections=0 \ +; RUN: -data-sections=0" > off.rsp + +;; Perform DTLTO with configuration state set. +; REDEFINE: %{extra_flags} = @on.rsp +; REDEFINE: %{distributor} = local.py +; REDEFINE: %{triple} = x86_64-unknown-linux-gnu +; RUN: %{command} +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=ON \ +; RUN: --implicit-check-not=-no-pgo-warn-mismatch +; ON-DAG: "-faddrsig" +; ON-DAG: "-ffunction-sections" +; ON-DAG: "-fdata-sections" + +;; Perform DTLTO with configuration state unset. +; REDEFINE: %{extra_flags} = @off.rsp +; REDEFINE: %{distributor} = local.py +; RUN: %{command} +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=OFF +; OFF-NOT: --implicit-check-not=--faddrsig +; OFF-NOT: --implicit-check-not=--ffunction-sections +; OFF-NOT: --implicit-check-not=--fdata-sections +; OFF-NOT: --implicit-check-not=-no-pgo-warn-mismatch + + +;; Check optimisation level. + +; RUN: llvm-lto2 run \ +; RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ +; RUN: @x86_64-unknown-linux-gnu.rsp \ +; RUN: -O3 + +; RUN: not llvm-lto2 run \ +; RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \ +; RUN: @x86_64-unknown-linux-gnu.rsp \ +; RUN: -O3 2>&1 | FileCheck %s --check-prefix=OPTLEVEL +; OPTLEVEL-DAG: "-O3" + + +;; Check relocation model. + +; REDEFINE: %{extra_flags} = -relocation-model=pic +; REDEFINE: %{distributor} = local.py +; RUN: %{command} +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=PIC +; PIC: -fpic + + +; REDEFINE: %{extra_flags} = -relocation-model=pic +; REDEFINE: %{distributor} = local.py +; REDEFINE: %{triple} = x86_64-pc-windows-msvc +; RUN: %{command} +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=NOPIC +; REDEFINE: %{triple} = x86_64-unknown-linux-gnu +; NOPIC-NOT: -fpic + +;; Check specifying a sample profile. +; REDEFINE: %{extra_flags} = --lto-sample-profile-file="missing.profdata" +; REDEFINE: %{distributor} = local.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE_ERR +; SAMPLE_PROFILE_ERR: no such file or directory: 'missing.profdata' +; REDEFINE: %{distributor} = validate.py +; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE +; SAMPLE_PROFILE-DAG: "-fprofile-sample-use=missing.profdata" + + +;--- x86_64-unknown-linux-gnu.ll + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @globalfunc1() { +entry: + ret void +} + +;--- x86_64-pc-windows-msvc.ll + +target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc" + +define void @globalfunc2() { +entry: + ret void +} diff --git a/cross-project-tests/dtlto/lit.local.cfg b/cross-project-tests/dtlto/lit.local.cfg new file mode 100644 index 0000000000000..9a5f7aff60249 --- /dev/null +++ b/cross-project-tests/dtlto/lit.local.cfg @@ -0,0 +1,2 @@ +if any(feature not in config.available_features for feature in ["clang", "llvm-lto2", "opt"]): + config.unsupported = True diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py index 66fdd63632885..3d27954c7545e 100644 --- a/cross-project-tests/lit.cfg.py +++ b/cross-project-tests/lit.cfg.py @@ -19,7 +19,7 @@ config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell) # suffixes: A list of file extensions to treat as test files. -config.suffixes = [".c", ".cl", ".cpp", ".m"] +config.suffixes = [".c", ".cl", ".cpp", ".m", ".ll"] # excludes: A list of directories to exclude from the testsuite. The 'Inputs' # subdirectories contain auxiliary inputs for various tests in their parent @@ -96,6 +96,9 @@ def get_required_attr(config, attr_name): if lldb_path is not None: config.available_features.add("lldb") +for tool in ["llvm-lto2", "opt"]: + if llvm_config.use_llvm_tool(tool): + config.available_features.add(tool) def configure_dexter_substitutions(): """Configure substitutions for host platform and return list of dependencies""" diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 3a9d57f69f718..fa0bc6f69f006 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -2291,6 +2291,62 @@ class OutOfProcessThinBackend : public CGThinBackend { return Error::success(); } + // Derive a set of Clang options that will be shared/common for all DTLTO + // backend compilations. We are intentionally minimal here as these options + // must remain synchronized with the behavior of Clang. DTLTO does not support + // all the features available with in-process LTO. More features are expected + // to be added over time. Users can specify Clang options directly if a + // feature is not supported. Note that explicitly specified options that imply + // additional input or output file dependencies must be communicated to the + // distribution system, potentially by setting extra options on the + // distributor program. + // TODO: If this strategy of deriving options proves insufficient, alternative + // approaches should be considered, such as: + // - A serialization/deserialization format for LTO configuration. + // - Modifying LLD to be the tool that performs the backend compilations. + void buildCommonRemoteOptToolOptions() { + const lto::Config &C = Conf; + auto &Ops = CodegenOptions; + llvm::Triple TT{Jobs.front().Triple}; + + Ops.push_back(Saver.save("-O" + Twine(C.OptLevel))); + + if (C.Options.EmitAddrsig) + Ops.push_back("-faddrsig"); + if (C.Options.FunctionSections) + Ops.push_back("-ffunction-sections"); + if (C.Options.DataSections) + Ops.push_back("-fdata-sections"); + + if (C.RelocModel == Reloc::PIC_) + // Clang doesn't have -fpic for all triples. + if (!TT.isOSBinFormatCOFF()) + Ops.push_back("-fpic"); + + // Turn on/off warnings about profile cfg mismatch (default on) + // --lto-pgo-warn-mismatch. + if (!C.PGOWarnMismatch) { + Ops.push_back("-mllvm"); + Ops.push_back("-no-pgo-warn-mismatch"); + } + + // Enable sample-based profile guided optimizations. + // Sample profile file path --lto-sample-profile=. + if (!C.SampleProfile.empty()) { + Ops.push_back( + Saver.save("-fprofile-sample-use=" + Twine(C.SampleProfile))); + AdditionalInputs.insert(C.SampleProfile); + } + + // We don't know which of options will be used by Clang. + Ops.push_back("-Wno-unused-command-line-argument"); + + // Forward any supplied options. + if (!ThinLTORemoteOptToolArgs.empty()) + for (auto &a : ThinLTORemoteOptToolArgs) + Ops.push_back(a); + } + // Generates a JSON file describing the backend compilations, for the // distributor. bool emitDistributorJson(StringRef DistributorJson) { @@ -2327,7 +2383,7 @@ class OutOfProcessThinBackend : public CGThinBackend { JOS.value(Array{"summary_index", "-fthinlto-index=", 0}); JOS.value(Saver.save("--target=" + Twine(Jobs.front().Triple))); - for (const auto &A : ThinLTORemoteOptToolArgs) + for (const auto &A : CodegenOptions) JOS.value(A); }); }); @@ -2391,6 +2447,8 @@ class OutOfProcessThinBackend : public CGThinBackend { return make_error(BCError + "all triples must be consistent", inconvertibleErrorCode()); + buildCommonRemoteOptToolOptions(); + SString JsonFile = sys::path::parent_path(LinkerOutputFile); sys::path::append(JsonFile, sys::path::stem(LinkerOutputFile) + "." + UID + ".dist-file.json"); diff --git a/llvm/test/ThinLTO/X86/dtlto/json.test b/llvm/test/ThinLTO/X86/dtlto/json.test index f6a78957cda33..a90d9ec041047 100644 --- a/llvm/test/ThinLTO/X86/dtlto/json.test +++ b/llvm/test/ThinLTO/X86/dtlto/json.test @@ -43,6 +43,9 @@ CHECK-NEXT: "-fthinlto-index=" CHECK-NEXT: 0 CHECK-NEXT: ] CHECK: "--target=x86_64-unknown-linux-gnu" +CHECK: "-O2", +CHECK: "-fpic" +CHECK: "-Wno-unused-command-line-argument" CHECK: "--rota1=10" CHECK: "--rota2=20" diff --git a/llvm/utils/dtlto/local.py b/llvm/utils/dtlto/local.py new file mode 100644 index 0000000000000..7be109061310c --- /dev/null +++ b/llvm/utils/dtlto/local.py @@ -0,0 +1,25 @@ +import subprocess +import sys +import json +from pathlib import Path + +if __name__ == "__main__": + # Load the DTLTO information from the input JSON file. + data = json.loads(Path(sys.argv[-1]).read_bytes()) + + # Iterate over the jobs and execute the codegen tool. + for job in data["jobs"]: + jobargs = [] + for arg in data["common"]["args"]: + if isinstance(arg, list): + # arg is a "template", into which an external filename is to be + # inserted. The first element of arg names an array of strings + # in the job. The remaining elements of arg are either indices + # into the array or literal strings. + files, rest = job[arg[0]], arg[1:] + jobargs.append( + "".join(files[x] if isinstance(x, int) else x for x in rest) + ) + else: + jobargs.append(arg) + subprocess.check_call(jobargs) From 2c9710fa9e8eaa52f70bd8987f0129cd522f9437 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 19 Feb 2025 04:11:17 +0000 Subject: [PATCH 08/22] [DTLTO][LLVM] Allow LTO to take an AddBuffer function and use in DTLTO Replace the use of AddStream in the DTLTO ThinLTO backend to add files to the link with AddBuffer. Unlike the InProcess ThinLTO backend, DTLTO runs the backend compilation jobs by invoking an external process (currently clang). This writes the output object file to disk. Therefore, DTLTO requires a performant way of adding an existing file to the link. Note that the AddBuffer mechanism is also used for adding a file to the link if there is a cache hit. --- llvm/include/llvm/LTO/LTO.h | 26 ++++++++---- llvm/include/llvm/Support/Caching.h | 3 +- llvm/lib/LTO/LTO.cpp | 65 ++++++++++++++--------------- llvm/tools/llvm-lto2/llvm-lto2.cpp | 2 +- 4 files changed, 51 insertions(+), 45 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index c776b57c1af82..764f60dfd8458 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -264,7 +264,7 @@ class ThinBackendProc { using ThinBackendFunction = std::function( const Config &C, ModuleSummaryIndex &CombinedIndex, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, FileCache Cache)>; + AddStreamFn AddStream, AddBufferFn AddBuffer, FileCache Cache)>; /// This type defines the behavior following the thin-link phase during ThinLTO. /// It encapsulates a backend function and a strategy for thread pool @@ -279,10 +279,10 @@ struct ThinBackend { std::unique_ptr operator()( const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, FileCache Cache) { + AddStreamFn AddStream, AddBufferFn AddBuffer, FileCache Cache) { assert(isValid() && "Invalid backend function"); return Func(Conf, CombinedIndex, ModuleToDefinedGVSummaries, - std::move(AddStream), std::move(Cache)); + std::move(AddStream), std::move(AddBuffer), std::move(Cache)); } ThreadPoolStrategy getParallelism() const { return Parallelism; } bool isValid() const { return static_cast(Func); } @@ -308,7 +308,7 @@ ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, /// This ThinBackend generates the index shards and then runs the individual /// backend jobs via an external process. It takes the same parameters as the /// InProcessThinBackend, however, these parameters only control the behavior -/// when generating the index files for the modules. Addtionally: +/// when generating the index files for the modules. Additionally: /// LinkerOutputFile is a string that should identify this LTO invocation in /// the context of a wider build. It's used for naming to aid the user in /// identifying activity related to a specific LTO invocation. @@ -402,15 +402,22 @@ class LTO { /// full description of tasks see LTOBackend.h. unsigned getMaxTasks() const; - /// Runs the LTO pipeline. This function calls the supplied AddStream - /// function to add native object files to the link. + /// Runs the LTO pipeline. This function calls the supplied AddStream or + /// AddBuffer function to add native object files to the link depending on + /// whether the files are streamed into memory or written to disk by the + /// backend. /// /// The Cache parameter is optional. If supplied, it will be used to cache /// native object files and add them to the link. /// - /// The client will receive at most one callback (via either AddStream or + /// The AddBuffer parameter is only required for DTLTO, currently. It is + /// optional to minimise the impact on current LTO users (DTLTO is not used + /// currently). + /// + /// The client will receive at most one callback (via AddStream, AddBuffer or /// Cache) for each task identifier. - Error run(AddStreamFn AddStream, FileCache Cache = {}); + Error run(AddStreamFn AddStream, FileCache Cache = {}, + AddBufferFn AddBuffer = nullptr); /// Static method that returns a list of libcall symbols that can be generated /// by LTO but might not be visible from bitcode symbol table. @@ -555,7 +562,8 @@ class LTO { StringRef Triple); Error runRegularLTO(AddStreamFn AddStream); - Error runThinLTO(AddStreamFn AddStream, FileCache Cache, + Error runThinLTO(AddStreamFn AddStream, AddBufferFn AddBuffer, + FileCache Cache, const DenseSet &GUIDPreservedSymbols); Error checkPartiallySplit(); diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h index cf45145619d95..8c3ea4f205d4c 100644 --- a/llvm/include/llvm/Support/Caching.h +++ b/llvm/include/llvm/Support/Caching.h @@ -84,7 +84,8 @@ struct FileCache { std::string CacheDirectoryPath; }; -/// This type defines the callback to add a pre-existing file (e.g. in a cache). +/// This type defines the callback to add a pre-existing file (e.g. in a cache +/// or created by a backend compilation run as a separate process). /// /// Buffer callbacks must be thread safe. using AddBufferFn = std::function GUIDPreservedSymbols; DenseMap GUIDPrevailingResolutions; @@ -1222,7 +1222,7 @@ Error LTO::run(AddStreamFn AddStream, FileCache Cache) { if (!Result) // This will reset the GlobalResolutions optional once done with it to // reduce peak memory before importing. - Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols); + Result = runThinLTO(AddStream, AddBuffer, Cache, GUIDPreservedSymbols); if (StatsFile) PrintStatisticsJSON(StatsFile->os()); @@ -1448,7 +1448,6 @@ Error ThinBackendProc::emitFiles( namespace { class CGThinBackend : public ThinBackendProc { protected: - AddStreamFn AddStream; DenseSet CfiFunctionDefs; DenseSet CfiFunctionDecls; bool ShouldEmitIndexFiles; @@ -1457,12 +1456,10 @@ class CGThinBackend : public ThinBackendProc { CGThinBackend( const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, lto::IndexWriteCallback OnWrite, - bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, - ThreadPoolStrategy ThinLTOParallelism) + lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, + bool ShouldEmitImportsFiles, ThreadPoolStrategy ThinLTOParallelism) : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism), - AddStream(std::move(AddStream)), ShouldEmitIndexFiles(ShouldEmitIndexFiles) { for (auto &Name : CombinedIndex.cfiFunctionDefs()) CfiFunctionDefs.insert( @@ -1475,6 +1472,7 @@ class CGThinBackend : public ThinBackendProc { class InProcessThinBackend : public CGThinBackend { protected: + AddStreamFn AddStream; FileCache Cache; public: @@ -1484,10 +1482,10 @@ class InProcessThinBackend : public CGThinBackend { const DenseMap &ModuleToDefinedGVSummaries, AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles) - : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, - AddStream, OnWrite, ShouldEmitIndexFiles, - ShouldEmitImportsFiles, ThinLTOParallelism), - Cache(std::move(Cache)) {} + : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite, + ShouldEmitIndexFiles, ShouldEmitImportsFiles, + ThinLTOParallelism), + AddStream(std::move(AddStream)), Cache(std::move(Cache)) {} virtual Error runThinLTOBackendThread( AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM, @@ -1755,7 +1753,7 @@ ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism, auto Func = [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, FileCache Cache) { + AddStreamFn AddStream, AddBufferFn /*AddBuffer*/, FileCache Cache) { return std::make_unique( Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, AddStream, Cache, OnWrite, ShouldEmitIndexFiles, @@ -1877,7 +1875,7 @@ ThinBackend lto::createWriteIndexesThinBackend( auto Func = [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, FileCache Cache) { + AddStreamFn AddStream, AddBufferFn AddBuffer, FileCache Cache) { return std::make_unique( Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix, NativeObjectPrefix, ShouldEmitImportsFiles, @@ -1886,7 +1884,8 @@ ThinBackend lto::createWriteIndexesThinBackend( return ThinBackend(Func, Parallelism); } -Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, +Error LTO::runThinLTO(AddStreamFn AddStream, AddBufferFn AddBuffer, + FileCache Cache, const DenseSet &GUIDPreservedSymbols) { LLVM_DEBUG(dbgs() << "Running ThinLTO\n"); ThinLTO.CombinedIndex.releaseTemporaryMemory(); @@ -2094,7 +2093,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, if (!CodeGenDataThinLTOTwoRounds) { std::unique_ptr BackendProc = ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, - AddStream, Cache); + AddStream, AddBuffer, Cache); return RunBackends(BackendProc.get()); } @@ -2201,6 +2200,8 @@ namespace { class OutOfProcessThinBackend : public CGThinBackend { using SString = SmallString<128>; + AddBufferFn AddBuffer; + BumpPtrAllocator Alloc; StringSaver Saver{Alloc}; @@ -2232,15 +2233,16 @@ class OutOfProcessThinBackend : public CGThinBackend { const Config &Conf, ModuleSummaryIndex &CombinedIndex, ThreadPoolStrategy ThinLTOParallelism, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, lto::IndexWriteCallback OnWrite, - bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, - StringRef LinkerOutputFile, StringRef RemoteOptTool, - StringRef Distributor, bool SaveTemps) - : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, - AddStream, OnWrite, ShouldEmitIndexFiles, - ShouldEmitImportsFiles, ThinLTOParallelism), - LinkerOutputFile(LinkerOutputFile), RemoteOptTool(RemoteOptTool), - DistributorPath(Distributor), SaveTemps(SaveTemps) {} + AddStreamFn AddStream, AddBufferFn AddBuffer, + lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, + bool ShouldEmitImportsFiles, StringRef LinkerOutputFile, + StringRef RemoteOptTool, StringRef Distributor, bool SaveTemps) + : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite, + ShouldEmitIndexFiles, ShouldEmitImportsFiles, + ThinLTOParallelism), + AddBuffer(std::move(AddBuffer)), LinkerOutputFile(LinkerOutputFile), + RemoteOptTool(RemoteOptTool), DistributorPath(Distributor), + SaveTemps(SaveTemps) {} virtual void setup(unsigned MaxTasks) override { UID = itostr(sys::Process::getProcessId()); @@ -2484,13 +2486,7 @@ class OutOfProcessThinBackend : public CGThinBackend { BCError + "cannot open native object file: " + Job.NativeObjectPath + ": " + ec.message(), inconvertibleErrorCode()); - std::unique_ptr umb = std::move(objFileMbOrErr.get()); - Expected> StreamOrErr = - AddStream(Job.Task, Job.ModuleID); - if (Error Err = StreamOrErr.takeError()) - report_fatal_error(std::move(Err)); - std::unique_ptr Stream = std::move(*StreamOrErr); - *Stream->OS << umb->getMemBufferRef().getBuffer(); + AddBuffer(Job.Task, Job.ModuleID, std::move(objFileMbOrErr.get())); } return Error::success(); @@ -2506,11 +2502,12 @@ ThinBackend lto::createOutOfProcessThinBackend( auto Func = [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, FileCache /*Cache*/) { + AddStreamFn AddStream, AddBufferFn AddBuffer, FileCache /*Cache*/) { return std::make_unique( Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, - AddStream, OnWrite, ShouldEmitIndexFiles, ShouldEmitImportsFiles, - LinkerOutputFile, RemoteOptTool, Distributor, SaveTemps); + AddStream, AddBuffer, OnWrite, ShouldEmitIndexFiles, + ShouldEmitImportsFiles, LinkerOutputFile, RemoteOptTool, + Distributor, SaveTemps); }; return ThinBackend(Func, Parallelism); } diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp index 82f2db58acd11..df6de93f6f236 100644 --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -476,7 +476,7 @@ static int run(int argc, char **argv) { Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer), "failed to create cache"); - check(Lto.run(AddStream, Cache), "LTO::run failed"); + check(Lto.run(AddStream, Cache, AddBuffer), "LTO::run failed"); return static_cast(HasErrors); } From 0c46c0c9fdf34df345524069c504351d4db25b47 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 19 Feb 2025 04:33:25 +0000 Subject: [PATCH 09/22] [DTLTO][LLVM][Doc] Add DTLTO documentation --- llvm/docs/DTLTO.rst | 228 +++++++++++++++++++++++++++++++++++++++ llvm/docs/UserGuides.rst | 6 ++ 2 files changed, 234 insertions(+) create mode 100644 llvm/docs/DTLTO.rst diff --git a/llvm/docs/DTLTO.rst b/llvm/docs/DTLTO.rst new file mode 100644 index 0000000000000..3fed25a922f5a --- /dev/null +++ b/llvm/docs/DTLTO.rst @@ -0,0 +1,228 @@ +=================== +DTLTO +=================== +.. contents:: + :local: + :depth: 2 + +.. toctree:: + :maxdepth: 1 + +Distributed ThinLTO (DTLTO) +=========================== + +Distributed ThinLTO (DTLTO) facilitates the distribution of backend ThinLTO +compilations via external distribution systems such as Incredibuild. + +The existing method of distributing ThinLTO compilations via separate thin-link, +backend compilation, and link steps often requires significant changes to the +user's build process to adopt, as it requires using a build system which can +handle the dynamic dependencies specified by the index files, such as Bazel. + +DTLTO eliminates this need by managing distribution internally within the LLD +linker during the traditional link step. This allows DTLTO to be used with any +build process that supports in-process ThinLTO. + +Limitations +----------- + +The current implementation of DTLTO has the following limitations: + +- The ThinLTO cache is not supported. +- Only ELF and COFF platforms are supported. +- Archives with bitcode members are not supported. +- Only a very limited set of LTO configurations are currently supported, e.g., + support for basic block sections is not currently available. + +Overview of Operation +--------------------- + +For each ThinLTO backend compilation job, LLD: + +1. Generates the required summary index shard. +2. Records a list of input and output files. +3. Constructs a Clang command line to perform the ThinLTO backend compilation. + +This information is supplied, via a JSON file, to a distributor program that +executes the backend compilations using a distribution system. Upon completion, +LLD integrates the compiled native object files into the link process. + +The design keeps the details of distribution systems out of the LLVM source +code. + +Distributors +------------ + +Distributors are programs responsible for: + +1. Consuming the JSON backend compilations job description file. +2. Translating job descriptions into requests for the distribution system. +3. Blocking execution until all backend compilations are complete. + +Distributors must return a non-zero exit code on failure. They can be +implemented as binaries or in scripting languages, such as Python. An example +script demonstrating basic local execution is available with the LLVM source +code. + +How Distributors Are Invoked +---------------------------- + +Clang and LLD provide options to specify a distributor program for managing +backend compilations. Distributor options and backend compilation options, can +also be specified. Such options are transparently forwarded. + +The backend compilations are currently performed by invoking Clang. For further +details, refer to: + +- Clang documentation: https://clang.llvm.org/docs/ThinLTO.html +- LLD documentation: https://lld.llvm.org/DTLTO.html + +When invoked with a distributor, LLD generates a JSON file describing the +backend compilation jobs and executes the distributor passing it this file. The +JSON file provides the following information to the distributor: + +- The **command line** to execute the backend compilations. + - DTLTO constructs a Clang command line by translating some of the LTO + configuration state into Clang options and forwarding options specified + by the user. + +- **Link output path**. + - A string identifying the output to which this LTO invocation will + contribute. Distributors can use this to label build jobs for informational + purposes. + +- **Linker's version string**. + - Distributors can use this to determine if the invoked remote optimisation + tool is compatible. + +- The list of **imports** required for each job. + - The per-job list of bitcode files from which importing will occur. This is + the same information that is emitted into import files for ThinLTO. + +- The **input files** required for each job. + - The per-job set of files required for backend compilation, such as bitcode + files, summary index files, and profile data. + +- The **output files** generated by each job. + - The per-job files generated by the backend compilations, such as compiled + object files and toolchain metrics. + +Temporary Files +--------------- + +During its operation, DTLTO generates temporary files. Temporary files are +created in the same directory as the linker's output file and their filenames +include the stem of the bitcode module, or the output file that the LTO +invocation is contributing to, to aid the user in identifying them: + +- **JSON Job Description File**: + - Format: `dtlto..dist-file.json` + - Example: `dtlto.77380.dist-file.json` (for output file `dtlto.elf`). + +- **Object Files From Backend Compilations**: + - Format: `...native.o` + - Example: `my.1.77380.native.o` (for bitcode module `my.o`). + +- **Summary Index Shard Files**: + - Format: `...native.o.thinlto.bc` + - Example: `my.1.77380.native.o.thinlto.bc` (for bitcode module `my.o`). + +Temporary files are removed, by default, after the backend compilations complete. + +JSON Schema +----------- + +Below is an example of a JSON job file for backend compilation of the module +`dtlto.o`: + +.. code-block:: json + + { + "common": { + "linker_output": "dtlto.elf", + "linker_version": "LLD 20.0.0", + "args": [ + "/usr/local/clang", + "-O3", "-fprofile-sample-use=my.profdata", + "-o", ["primary_output", 0], + "-c", "-x", "ir", ["primary_input", 0], + ["summary_index", "-fthinlto-index=", 0], + "--target=x86_64-sie-ps5" + ] + }, + "jobs": [ + { + "primary_input": ["dtlto.o"], + "summary_index": ["dtlto.1.51232.native.o.thinlto.bc"], + "primary_output": ["dtlto.1.51232.native.o"], + "imports": [], + "additional_inputs": ["my.profdata"] + } + ] + } + +Key Features of the Schema +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- **Input/Output Paths**: Paths are stored in per-file-type array fields. This + allows files to be adjusted, if required, to meet the constraints of the + underlying distribution system. For example, a system may only be able to read + and write remote files to `C:\\sandbox`. The remote paths used can be adjusted + by the distributor for such constraints. Once outputs are back on the local + system, the distributor can rename them as required. + + +- **Command-Line Template**: Command-line options are stored in a common + template to avoid duplication for each job. The template consists of an array + of strings and arrays. The arrays are placeholders which reference per-job + paths. This allows the remote optimisation tool to be changed without updating + the distributors. + +Command-Line Expansion Example +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To create the backend compilation commands, the command-line template is +expanded for each job. Placeholders are expanded in the following way: The first +array element specifies the name of the array field to look in. The remaining +elements are converted to strings and concatenated. Integers are converted by +indexing into the specified array. + +The example above generates the following backend compilation command for +`main.o`: + +.. code-block:: console + + /usr/local/clang -O3 -fprofile-sample-use=my.profdata \ + -o dtlto.1.51232.native.o -c -x ir dtlto.o \ + -fthinlto-index=dtlto.1.51232.native.o.thinlto.bc --target=x86_64-sie-ps5 + +This expansion scheme allows the remote optimization tool to be changed without +updating the distributors. For example, if the "args" field in the above example +was replaced with: + +.. code-block:: json + + "args": [ + "custom-codgen-tool", + "-opt-level=2", + "-profile-instrument-use-path=my.profdata", + "-output", ["primary_output", 0], + "-input", ["primary_input", 0], + "-thinlto-index", ["summary_index", 0], + "-triple", "x86_64-sie-ps5" + ] + +Then distributors can expand the command line without needing to be updated: + +.. code-block:: console + + custom-codgen-tool -opt-level=2 -profile-instrument-use-path=my.profdata \ + -output dtlto.1.51232.native.o -input dtlto.o \ + -thinlto-index dtlto.1.51232.native.o.thinlto.bc -triple x86_64-sie-ps5 + +Constraints +----------- + +- Matching versions of Clang and LLD should be used. +- The distributor used must support the JSON schema generated by the version of + LLD in use. \ No newline at end of file diff --git a/llvm/docs/UserGuides.rst b/llvm/docs/UserGuides.rst index 6eee564713d6d..3e16fe42b7d11 100644 --- a/llvm/docs/UserGuides.rst +++ b/llvm/docs/UserGuides.rst @@ -32,6 +32,7 @@ intermediate LLVM representation. DebuggingJITedCode DirectXUsage Docker + DTLTO FatLTO ExtendingLLVM GitHub @@ -164,6 +165,11 @@ Optimizations This document describes the interface between LLVM intermodular optimizer and the linker and its design +:doc:`DTLTO` + This document describes the DTLTO implementation, which allows for + distributing ThinLTO backend compilations without requiring support from + the build system. + :doc:`GoldPlugin` How to build your programs with link-time optimization on Linux. From 33dbf558840b096c75f84dca6b2741674d50164c Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 19 Feb 2025 14:28:30 +0000 Subject: [PATCH 10/22] [DTLTO][LLVM] clang format LTO.h to prevent automated checks errors Some of the code in LTO.h did not conform to the LLVM coding standard. However, it did match the style already used in that file. However this was causing automated code-formatting checks from Github to fail which was confusing on the PR. I decided to apply clang-format everywhere to prevent this - even though the new code no longer matches the style of the existing. --- llvm/include/llvm/LTO/LTO.h | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 764f60dfd8458..8e6d638602317 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -318,14 +318,11 @@ ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, /// jobs execution. /// SaveTemps is a debugging tool that prevents temporary files created by this /// backend from being cleaned up. -ThinBackend createOutOfProcessThinBackend(ThreadPoolStrategy Parallelism, - IndexWriteCallback OnWrite, - bool ShouldEmitIndexFiles, - bool ShouldEmitImportsFiles, - StringRef LinkerOutputFile, - StringRef RemoteOptTool, - StringRef Distributor, - bool SaveTemps); +ThinBackend createOutOfProcessThinBackend( + ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite, + bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, + StringRef LinkerOutputFile, StringRef RemoteOptTool, StringRef Distributor, + bool SaveTemps); /// This ThinBackend writes individual module indexes to files, instead of /// running the individual backend jobs. This backend is for distributed builds From 9b5162d1d4c74f3807c5f6cb92fd337c3b4f7a15 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Thu, 20 Feb 2025 23:06:06 +0000 Subject: [PATCH 11/22] Improve the test distributors 1. Use "with" when opening files. 2. Add module docstrings. --- llvm/test/ThinLTO/X86/dtlto/dtlto.test | 18 +++++++++--------- llvm/test/ThinLTO/X86/dtlto/imports.test | 3 +-- llvm/test/ThinLTO/X86/dtlto/summary.test | 2 +- llvm/test/ThinLTO/X86/dtlto/triple.test | 16 ++++++++-------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/llvm/test/ThinLTO/X86/dtlto/dtlto.test b/llvm/test/ThinLTO/X86/dtlto/dtlto.test index 17dde95c022d8..b3198bdd0dd4a 100644 --- a/llvm/test/ThinLTO/X86/dtlto/dtlto.test +++ b/llvm/test/ThinLTO/X86/dtlto/dtlto.test @@ -14,15 +14,15 @@ RUN: opt t2.ll -o t2.o RUN: mkdir %t/out && cd %t/out # Define a substitution to share the common DTLTO arguments. -DEFINE: %{command} =llvm-lto2 run ../t1.bc ../t2.bc -o t.o \ -DEFINE: -dtlto \ -DEFINE: -dtlto-remote-opt-tool=dummy \ -DEFINE: -dtlto-distributor=%python \ -DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ -DEFINE: -thinlto-distributor-arg=../t1.o \ -DEFINE: -thinlto-distributor-arg=../t2.o \ -DEFINE: -r=../t1.bc,t1,px \ -DEFINE: -r=../t2.bc,t2,px +DEFINE: %{command} = llvm-lto2 run ../t1.bc ../t2.bc -o t.o \ +DEFINE: -dtlto \ +DEFINE: -dtlto-remote-opt-tool=dummy \ +DEFINE: -dtlto-distributor=%python \ +DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +DEFINE: -thinlto-distributor-arg=../t1.o \ +DEFINE: -thinlto-distributor-arg=../t2.o \ +DEFINE: -r=../t1.bc,t1,px \ +DEFINE: -r=../t2.bc,t2,px # Perform DTLTO. mock.py does not do any compilation, instead it simply writes # the contents of the object files supplied on the command line into the diff --git a/llvm/test/ThinLTO/X86/dtlto/imports.test b/llvm/test/ThinLTO/X86/dtlto/imports.test index 57ace00aeaf9e..b167525eaee9e 100644 --- a/llvm/test/ThinLTO/X86/dtlto/imports.test +++ b/llvm/test/ThinLTO/X86/dtlto/imports.test @@ -8,8 +8,7 @@ RUN: opt -thinlto-bc 1.ll -o 1.bc -O2 # Define a substitution to share the common DTLTO arguments. Note that the use # of validate.py will cause a failure as it does not create output files. -DEFINE: %{command} =llvm-lto2 run \ -DEFINE: 0.bc 1.bc -o t.o \ +DEFINE: %{command} = llvm-lto2 run 0.bc 1.bc -o t.o \ DEFINE: -dtlto \ DEFINE: -dtlto-remote-opt-tool=dummy \ DEFINE: -dtlto-distributor=%python \ diff --git a/llvm/test/ThinLTO/X86/dtlto/summary.test b/llvm/test/ThinLTO/X86/dtlto/summary.test index dab59ece12f8e..98f1039c889fe 100644 --- a/llvm/test/ThinLTO/X86/dtlto/summary.test +++ b/llvm/test/ThinLTO/X86/dtlto/summary.test @@ -12,7 +12,7 @@ RUN: opt t1.ll -o t1.o RUN: opt t2.ll -o t2.o # Define a substitution to share the common arguments. -DEFINE: %{command} =llvm-lto2 run t1.bc t2.bc -o t.o \ +DEFINE: %{command} = llvm-lto2 run t1.bc t2.bc -o t.o \ DEFINE: -r=t1.bc,t1,px \ DEFINE: -r=t2.bc,t2,px \ DEFINE: -r=t2.bc,t1 \ diff --git a/llvm/test/ThinLTO/X86/dtlto/triple.test b/llvm/test/ThinLTO/X86/dtlto/triple.test index f0642281535a9..5c0a834dbb1c3 100644 --- a/llvm/test/ThinLTO/X86/dtlto/triple.test +++ b/llvm/test/ThinLTO/X86/dtlto/triple.test @@ -13,14 +13,14 @@ RUN: opt t2.ll -o t2.o # Perform DTLTO. mock.py does not do any compilation, instead it emits the # object files supplied using -thinlto-distributor-arg in job order. RUN: not llvm-lto2 run t1.bc t2.bc -o t.o -save-temps \ -RUN: -dtlto \ -RUN: -dtlto-remote-opt-tool=dummy \ -RUN: -dtlto-distributor=%python \ -RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ -RUN: -thinlto-distributor-arg=t1.o \ -RUN: -thinlto-distributor-arg=t2.o \ -RUN: -r=t1.bc,t1,px \ -RUN: -r=t2.bc,t2,px 2>&1 | FileCheck %s +RUN: -dtlto \ +RUN: -dtlto-remote-opt-tool=dummy \ +RUN: -dtlto-distributor=%python \ +RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: -thinlto-distributor-arg=t1.o \ +RUN: -thinlto-distributor-arg=t2.o \ +RUN: -r=t1.bc,t1,px \ +RUN: -r=t2.bc,t2,px 2>&1 | FileCheck %s # This check ensures that we have failed for the expected reason. CHECK: failed: DTLTO backend compilation: all triples must be consistent From 2be4b0c1d909d88b3474770ad53701a1fc1fcc49 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Thu, 20 Feb 2025 23:20:11 +0000 Subject: [PATCH 12/22] Address minor test nits --- .../dtlto/dtlto-translate-options.ll | 12 ++++----- llvm/utils/dtlto/local.py | 18 ++++++++++++- llvm/utils/dtlto/mock.py | 24 ++++++++++++++++- llvm/utils/dtlto/validate.py | 26 ++++++++++++++++--- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/cross-project-tests/dtlto/dtlto-translate-options.ll b/cross-project-tests/dtlto/dtlto-translate-options.ll index bbb6ccf33fe7c..9e95c52632125 100644 --- a/cross-project-tests/dtlto/dtlto-translate-options.ll +++ b/cross-project-tests/dtlto/dtlto-translate-options.ll @@ -8,8 +8,8 @@ ; RUN: opt -thinlto-bc x86_64-pc-windows-msvc.ll -o x86_64-pc-windows-msvc.bc -;; Check that any invalid arguments would cause a Clang error. This property is -;; relied on by the actual testcases later in this test. +;; Check that invalid arguments cause a Clang error. This property is relied on +;; by the actual testcases later in this test. ; RUN: not %clang -x ir x86_64-unknown-linux-gnu.ll \ ; RUN: -invalid-incorrect-not-an-option 2>&1 | FileCheck %s --check-prefix=SANITY1 ; SANITY1: unknown argument: '-invalid-incorrect-not-an-option' @@ -25,7 +25,7 @@ ; DEFINE: @%{triple}.rsp %{extra_flags} -;; Write common arguments to a response files. +;; Write common arguments to response files. ; RUN: echo "x86_64-unknown-linux-gnu.bc -o x86_64-unknown-linux-gnu.o \ ; RUN: -dtlto \ @@ -63,7 +63,7 @@ ; RUN: %{command} ; REDEFINE: %{distributor} = validate.py ; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=ON \ -; RUN: --implicit-check-not=-no-pgo-warn-mismatch +; RUN: --implicit-check-not=-no-pgo-warn-mismatch ; ON-DAG: "-faddrsig" ; ON-DAG: "-ffunction-sections" ; ON-DAG: "-fdata-sections" @@ -80,7 +80,7 @@ ; OFF-NOT: --implicit-check-not=-no-pgo-warn-mismatch -;; Check optimisation level. +;; Check optimization level. ; RUN: llvm-lto2 run \ ; RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ @@ -124,7 +124,6 @@ ;--- x86_64-unknown-linux-gnu.ll - target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" @@ -134,7 +133,6 @@ entry: } ;--- x86_64-pc-windows-msvc.ll - target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-windows-msvc" diff --git a/llvm/utils/dtlto/local.py b/llvm/utils/dtlto/local.py index 7be109061310c..109cbf18ff111 100644 --- a/llvm/utils/dtlto/local.py +++ b/llvm/utils/dtlto/local.py @@ -1,3 +1,18 @@ +""" +DTLTO local serial distributor. + +This script parses the Distributed ThinLTO (DTLTO) JSON file and serially +executes the specified code generation tool on the local host to perform each +backend compilation job. This simple functional distributor is intended to be +used for integration tests. + +Usage: + python dtlto_codegen.py + +Arguments: + - : JSON file describing the DTLTO jobs. +""" + import subprocess import sys import json @@ -5,7 +20,8 @@ if __name__ == "__main__": # Load the DTLTO information from the input JSON file. - data = json.loads(Path(sys.argv[-1]).read_bytes()) + with Path(sys.argv[-1]).open() as f: + data = json.load(f) # Iterate over the jobs and execute the codegen tool. for job in data["jobs"]: diff --git a/llvm/utils/dtlto/mock.py b/llvm/utils/dtlto/mock.py index 423a7f7beb194..d7dad83eeaaaf 100644 --- a/llvm/utils/dtlto/mock.py +++ b/llvm/utils/dtlto/mock.py @@ -1,3 +1,24 @@ +""" +DTLTO Mock Distributor. + +This script acts as a mock distributor for Distributed ThinLTO (DTLTO). It is +used for testing DTLTO when a Clang binary is not be available to invoke to +perform the backend compilation jobs. + +Usage: + python mock_distributor.py ... + +Arguments: + - , , ... : Input files to be copied. + - : JSON file describing the DTLTO jobs. + +The script performs the following: + 1. Reads the JSON file containing job descriptions. + 2. For each job copies the corresponding input file to the output location + specified for that job. + 3. Validates the JSON format using the `validate` module. +""" + import sys import json import shutil @@ -9,7 +30,8 @@ distributor_args = sys.argv[1:-1] # Load the DTLTO information from the input JSON file. - data = json.loads(Path(json_arg).read_bytes()) + with Path(json_arg).open() as f: + data = json.load(f) # Iterate over the jobs and create the output # files by copying over the supplied input files. diff --git a/llvm/utils/dtlto/validate.py b/llvm/utils/dtlto/validate.py index eb14f2c746a8e..3fa13bebdf17f 100644 --- a/llvm/utils/dtlto/validate.py +++ b/llvm/utils/dtlto/validate.py @@ -1,3 +1,22 @@ +""" +DTLTO JSON Validator. + +This script is used for DTLTO testing to check that the distributor has +been invoked correctly. + +Usage: + python validate.py + +Arguments: + - : JSON file describing the DTLTO jobs. + +The script does the following: + 1. Prints the supplied CLI arguments. + 2. Loads the JSON file. + 3. Validates the structure and required fields. + 4. Pretty prints the JSON. +""" + import sys import json from pathlib import Path @@ -64,16 +83,17 @@ def validate_reference(a): if __name__ == "__main__": - json_arg = sys.argv[-1] + json_arg = Path(sys.argv[-1]) distributor_args = sys.argv[1:-1] print(f"{distributor_args=}") # Load the DTLTO information from the input JSON file. - jdoc = json.loads(Path(json_arg).read_bytes()) + with json_arg.open() as f: + jdoc = json.load(f) # Write the input JSON to stdout. print(json.dumps(jdoc, indent=4)) - # Check the format of the JSON + # Check the format of the JSON. validate(jdoc) From 892348406e07e1d2af8056061adaaa09af770c64 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Tue, 25 Feb 2025 00:15:34 +0000 Subject: [PATCH 13/22] Support more than one LTO partition. --- llvm/include/llvm/LTO/LTO.h | 2 +- llvm/lib/LTO/LTO.cpp | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 8e6d638602317..f400e0d6deb39 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -225,7 +225,7 @@ class ThinBackendProc { BackendThreadPool(ThinLTOParallelism) {} virtual ~ThinBackendProc() = default; - virtual void setup(unsigned MaxTasks) {} + virtual void setup(unsigned MaxTasks, unsigned ReservedTasks) {} virtual Error start( unsigned Task, BitcodeModule BM, const FunctionImporter::ImportMapTy &ImportList, diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 6ad31d68578cb..3873cdf8cf768 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -2062,7 +2062,8 @@ Error LTO::runThinLTO(AddStreamFn AddStream, AddBufferFn AddBuffer, ResolvedODR[Mod.first], ThinLTO.ModuleMap, ThinLTO.ModuleTriples); }; - BackendProcess->setup(ModuleMap.size()); + BackendProcess->setup(ModuleMap.size(), + RegularLTO.ParallelCodeGenParallelismLevel); if (BackendProcess->getThreadCount() == 1 || BackendProcess->isSensitiveToInputOrder()) { @@ -2228,6 +2229,9 @@ class OutOfProcessThinBackend : public CGThinBackend { // A unique string to identify the current link. SmallString<8> UID; + // The first ReservedTasks entries in the task range are used for Full LTO. + unsigned ReservedTasks; + public: OutOfProcessThinBackend( const Config &Conf, ModuleSummaryIndex &CombinedIndex, @@ -2244,9 +2248,10 @@ class OutOfProcessThinBackend : public CGThinBackend { RemoteOptTool(RemoteOptTool), DistributorPath(Distributor), SaveTemps(SaveTemps) {} - virtual void setup(unsigned MaxTasks) override { + virtual void setup(unsigned MaxTasks, unsigned ReservedTasks) override { UID = itostr(sys::Process::getProcessId()); Jobs.resize((size_t)MaxTasks); + this->ReservedTasks = ReservedTasks; } Error start( @@ -2263,7 +2268,7 @@ class OutOfProcessThinBackend : public CGThinBackend { sys::path::append(ObjFilePath, sys::path::stem(ModulePath) + "." + itostr(Task) + "." + UID + ".native.o"); - Job &J = Jobs[Task - 1]; /*Task 0 is reserved*/ + Job &J = Jobs[Task - ReservedTasks]; J = {Task, ModulePath, ModuleTriples[ModulePath], From 0bfafdd4630c71fdb845760172b863ddc5d5c650 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Thu, 27 Feb 2025 23:25:49 +0000 Subject: [PATCH 14/22] UI improvements to the current `remote-opt-tool` - Replace `thinlto-remote-opt-tool` with `thinlto-remote-compiler` - Make `thinlto-remote-compiler` a cl::opt --- .../dtlto/dtlto-translate-options.ll | 12 +++---- llvm/include/llvm/LTO/LTO.h | 5 +-- llvm/lib/LTO/LTO.cpp | 33 ++++++++++--------- llvm/test/ThinLTO/X86/dtlto/dtlto.test | 1 - llvm/test/ThinLTO/X86/dtlto/imports.test | 1 - llvm/test/ThinLTO/X86/dtlto/json.test | 6 ++-- llvm/test/ThinLTO/X86/dtlto/summary.test | 1 - llvm/test/ThinLTO/X86/dtlto/triple.test | 1 - llvm/tools/llvm-lto2/llvm-lto2.cpp | 6 +--- 9 files changed, 28 insertions(+), 38 deletions(-) diff --git a/cross-project-tests/dtlto/dtlto-translate-options.ll b/cross-project-tests/dtlto/dtlto-translate-options.ll index 9e95c52632125..7f5888cf9e6c7 100644 --- a/cross-project-tests/dtlto/dtlto-translate-options.ll +++ b/cross-project-tests/dtlto/dtlto-translate-options.ll @@ -21,7 +21,7 @@ ; DEFINE: %{triple} = dummy ; DEFINE: %{command} = llvm-lto2 run \ ; DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/%{distributor} \ -; DEFINE: -thinlto-remote-opt-tool-arg=-Wunused-command-line-argument \ +; DEFINE: -thinlto-remote-compiler-arg=-Wunused-command-line-argument \ ; DEFINE: @%{triple}.rsp %{extra_flags} @@ -29,17 +29,17 @@ ; RUN: echo "x86_64-unknown-linux-gnu.bc -o x86_64-unknown-linux-gnu.o \ ; RUN: -dtlto \ -; RUN: -dtlto-remote-opt-tool=%clang \ -; RUN: -thinlto-remote-opt-tool-arg=-Werror \ ; RUN: -dtlto-distributor=%python \ +; RUN: -thinlto-remote-compiler=%clang \ +; RUN: -thinlto-remote-compiler-arg=-Werror \ ; RUN: -r=x86_64-unknown-linux-gnu.bc,globalfunc1,plx" > x86_64-unknown-linux-gnu.rsp ; RUN: echo "x86_64-pc-windows-msvc.bc -o x86_64-pc-windows-msvc.o \ ; RUN: -dtlto \ -; RUN: -dtlto-remote-opt-tool=%clang \ -; RUN: -thinlto-remote-opt-tool-arg=-Werror \ -; RUN: -thinlto-remote-opt-tool-arg=-Wno-override-module \ ; RUN: -dtlto-distributor=%python \ +; RUN: -thinlto-remote-compiler=%clang \ +; RUN: -thinlto-remote-compiler-arg=-Werror \ +; RUN: -thinlto-remote-compiler-arg=-Wno-override-module \ ; RUN: -r=x86_64-pc-windows-msvc.bc,globalfunc2,plx" > x86_64-pc-windows-msvc.rsp diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index f400e0d6deb39..b7460b97a47e6 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -312,8 +312,6 @@ ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, /// LinkerOutputFile is a string that should identify this LTO invocation in /// the context of a wider build. It's used for naming to aid the user in /// identifying activity related to a specific LTO invocation. -/// RemoteOptTool specifies the path to a Clang executable to be invoked for the -/// backend jobs. /// Distributor specifies the path to a process to invoke to manage the backend /// jobs execution. /// SaveTemps is a debugging tool that prevents temporary files created by this @@ -321,8 +319,7 @@ ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, ThinBackend createOutOfProcessThinBackend( ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, - StringRef LinkerOutputFile, StringRef RemoteOptTool, StringRef Distributor, - bool SaveTemps); + StringRef LinkerOutputFile, StringRef Distributor, bool SaveTemps); /// This ThinBackend writes individual module indexes to files, instead of /// running the individual backend jobs. This backend is for distributed builds diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 3873cdf8cf768..724bcac7a2771 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -99,10 +99,15 @@ cl::list AdditionalThinLTODistributorArgs( "thinlto-distributor-arg", cl::desc("Additional arguments to pass to the ThinLTO distributor")); +cl::opt + ThinLTORemoteCompiler("thinlto-remote-compiler", + cl::desc("Additional arguments to pass to the " + "ThinLTO remote optimization tool")); + cl::list - ThinLTORemoteOptToolArgs("thinlto-remote-opt-tool-arg", - cl::desc("Additional arguments to pass to the " - "ThinLTO remote optimization tool")); + ThinLTORemoteCompilerArgs("thinlto-remote-compiler-arg", + cl::desc("Additional arguments to pass to the " + "ThinLTO remote compiler")); } // namespace llvm // Computes a unique hash for the Module considering the current list of @@ -2207,7 +2212,6 @@ class OutOfProcessThinBackend : public CGThinBackend { StringSaver Saver{Alloc}; SString LinkerOutputFile; - SString RemoteOptTool; SString DistributorPath; bool SaveTemps; @@ -2240,13 +2244,12 @@ class OutOfProcessThinBackend : public CGThinBackend { AddStreamFn AddStream, AddBufferFn AddBuffer, lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, StringRef LinkerOutputFile, - StringRef RemoteOptTool, StringRef Distributor, bool SaveTemps) + StringRef Distributor, bool SaveTemps) : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries, OnWrite, ShouldEmitIndexFiles, ShouldEmitImportsFiles, ThinLTOParallelism), AddBuffer(std::move(AddBuffer)), LinkerOutputFile(LinkerOutputFile), - RemoteOptTool(RemoteOptTool), DistributorPath(Distributor), - SaveTemps(SaveTemps) {} + DistributorPath(Distributor), SaveTemps(SaveTemps) {} virtual void setup(unsigned MaxTasks, unsigned ReservedTasks) override { UID = itostr(sys::Process::getProcessId()); @@ -2311,7 +2314,7 @@ class OutOfProcessThinBackend : public CGThinBackend { // approaches should be considered, such as: // - A serialization/deserialization format for LTO configuration. // - Modifying LLD to be the tool that performs the backend compilations. - void buildCommonRemoteOptToolOptions() { + void buildCommonRemoteCompilerOptions() { const lto::Config &C = Conf; auto &Ops = CodegenOptions; llvm::Triple TT{Jobs.front().Triple}; @@ -2349,8 +2352,8 @@ class OutOfProcessThinBackend : public CGThinBackend { Ops.push_back("-Wno-unused-command-line-argument"); // Forward any supplied options. - if (!ThinLTORemoteOptToolArgs.empty()) - for (auto &a : ThinLTORemoteOptToolArgs) + if (!ThinLTORemoteCompilerArgs.empty()) + for (auto &a : ThinLTORemoteCompilerArgs) Ops.push_back(a); } @@ -2372,7 +2375,7 @@ class OutOfProcessThinBackend : public CGThinBackend { // Common command line template. JOS.attributeArray("args", [&]() { - JOS.value(RemoteOptTool); + JOS.value(ThinLTORemoteCompiler); // Reference to Job::NativeObjectPath. JOS.value("-o"); @@ -2454,7 +2457,7 @@ class OutOfProcessThinBackend : public CGThinBackend { return make_error(BCError + "all triples must be consistent", inconvertibleErrorCode()); - buildCommonRemoteOptToolOptions(); + buildCommonRemoteCompilerOptions(); SString JsonFile = sys::path::parent_path(LinkerOutputFile); sys::path::append(JsonFile, sys::path::stem(LinkerOutputFile) + "." + UID + @@ -2502,8 +2505,7 @@ class OutOfProcessThinBackend : public CGThinBackend { ThinBackend lto::createOutOfProcessThinBackend( ThreadPoolStrategy Parallelism, lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles, - StringRef LinkerOutputFile, StringRef RemoteOptTool, StringRef Distributor, - bool SaveTemps) { + StringRef LinkerOutputFile, StringRef Distributor, bool SaveTemps) { auto Func = [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap &ModuleToDefinedGVSummaries, @@ -2511,8 +2513,7 @@ ThinBackend lto::createOutOfProcessThinBackend( return std::make_unique( Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, AddStream, AddBuffer, OnWrite, ShouldEmitIndexFiles, - ShouldEmitImportsFiles, LinkerOutputFile, RemoteOptTool, - Distributor, SaveTemps); + ShouldEmitImportsFiles, LinkerOutputFile, Distributor, SaveTemps); }; return ThinBackend(Func, Parallelism); } diff --git a/llvm/test/ThinLTO/X86/dtlto/dtlto.test b/llvm/test/ThinLTO/X86/dtlto/dtlto.test index b3198bdd0dd4a..2239cdcc6bdf1 100644 --- a/llvm/test/ThinLTO/X86/dtlto/dtlto.test +++ b/llvm/test/ThinLTO/X86/dtlto/dtlto.test @@ -16,7 +16,6 @@ RUN: mkdir %t/out && cd %t/out # Define a substitution to share the common DTLTO arguments. DEFINE: %{command} = llvm-lto2 run ../t1.bc ../t2.bc -o t.o \ DEFINE: -dtlto \ -DEFINE: -dtlto-remote-opt-tool=dummy \ DEFINE: -dtlto-distributor=%python \ DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ DEFINE: -thinlto-distributor-arg=../t1.o \ diff --git a/llvm/test/ThinLTO/X86/dtlto/imports.test b/llvm/test/ThinLTO/X86/dtlto/imports.test index b167525eaee9e..d03ac291fd2b7 100644 --- a/llvm/test/ThinLTO/X86/dtlto/imports.test +++ b/llvm/test/ThinLTO/X86/dtlto/imports.test @@ -10,7 +10,6 @@ RUN: opt -thinlto-bc 1.ll -o 1.bc -O2 # of validate.py will cause a failure as it does not create output files. DEFINE: %{command} = llvm-lto2 run 0.bc 1.bc -o t.o \ DEFINE: -dtlto \ -DEFINE: -dtlto-remote-opt-tool=dummy \ DEFINE: -dtlto-distributor=%python \ DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \ DEFINE: -thinlto-distributor-arg=0.bc \ diff --git a/llvm/test/ThinLTO/X86/dtlto/json.test b/llvm/test/ThinLTO/X86/dtlto/json.test index a90d9ec041047..ee15dcfff1282 100644 --- a/llvm/test/ThinLTO/X86/dtlto/json.test +++ b/llvm/test/ThinLTO/X86/dtlto/json.test @@ -11,11 +11,11 @@ RUN: opt -thinlto-bc t2.ll -o t2.bc RUN: not llvm-lto2 run t1.bc t2.bc -o my.output \ RUN: -r=t1.bc,t1,px -r=t2.bc,t2,px \ RUN: -dtlto \ -RUN: -dtlto-remote-opt-tool=my_clang.exe \ RUN: -dtlto-distributor=%python \ RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \ -RUN: -thinlto-remote-opt-tool-arg=--rota1=10 \ -RUN: -thinlto-remote-opt-tool-arg=--rota2=20 \ +RUN: -thinlto-remote-compiler=my_clang.exe \ +RUN: -thinlto-remote-compiler-arg=--rota1=10 \ +RUN: -thinlto-remote-compiler-arg=--rota2=20 \ RUN: -thinlto-distributor-arg=--da1=10 \ RUN: -thinlto-distributor-arg=--da2=10 \ RUN: 2>&1 | FileCheck %s diff --git a/llvm/test/ThinLTO/X86/dtlto/summary.test b/llvm/test/ThinLTO/X86/dtlto/summary.test index 98f1039c889fe..a2fb08a736968 100644 --- a/llvm/test/ThinLTO/X86/dtlto/summary.test +++ b/llvm/test/ThinLTO/X86/dtlto/summary.test @@ -20,7 +20,6 @@ DEFINE: -thinlto-emit-indexes # Perform DTLTO. RUN: %{command} -dtlto \ -RUN: -dtlto-remote-opt-tool=dummy \ RUN: -dtlto-distributor=%python \ RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ RUN: -thinlto-distributor-arg=t1.o \ diff --git a/llvm/test/ThinLTO/X86/dtlto/triple.test b/llvm/test/ThinLTO/X86/dtlto/triple.test index 5c0a834dbb1c3..de6cedd382dd7 100644 --- a/llvm/test/ThinLTO/X86/dtlto/triple.test +++ b/llvm/test/ThinLTO/X86/dtlto/triple.test @@ -14,7 +14,6 @@ RUN: opt t2.ll -o t2.o # object files supplied using -thinlto-distributor-arg in job order. RUN: not llvm-lto2 run t1.bc t2.bc -o t.o -save-temps \ RUN: -dtlto \ -RUN: -dtlto-remote-opt-tool=dummy \ RUN: -dtlto-distributor=%python \ RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ RUN: -thinlto-distributor-arg=t1.o \ diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp index df6de93f6f236..f00af52b6ca5f 100644 --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -99,10 +99,6 @@ static cl::opt static cl::opt DTLTO("dtlto", cl::desc("Perform DTLTO")); -static cl::opt - DTLTORemoteOptTool("dtlto-remote-opt-tool", - cl::desc("Specify the remote opt tool for DTLTO")); - static cl::opt DTLTODistributor("dtlto-distributor", cl::desc("Specify the distributor for DTLTO")); @@ -372,7 +368,7 @@ static int run(int argc, char **argv) { Backend = createOutOfProcessThinBackend( llvm::heavyweight_hardware_concurrency(Threads), /*OnWrite=*/{}, ThinLTOEmitIndexes, ThinLTOEmitImports, OutputFilename, - DTLTORemoteOptTool, DTLTODistributor, SaveTemps); + DTLTODistributor, SaveTemps); } else Backend = createInProcessThinBackend( llvm::heavyweight_hardware_concurrency(Threads), From 2f9d3817b0bdadc4daec7877b575c0e9c5e51c6b Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Fri, 28 Feb 2025 12:25:01 +0000 Subject: [PATCH 15/22] Update python module docstrings to match the script names --- llvm/utils/dtlto/local.py | 2 +- llvm/utils/dtlto/mock.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/utils/dtlto/local.py b/llvm/utils/dtlto/local.py index 109cbf18ff111..adf463c02f479 100644 --- a/llvm/utils/dtlto/local.py +++ b/llvm/utils/dtlto/local.py @@ -7,7 +7,7 @@ used for integration tests. Usage: - python dtlto_codegen.py + python local.py Arguments: - : JSON file describing the DTLTO jobs. diff --git a/llvm/utils/dtlto/mock.py b/llvm/utils/dtlto/mock.py index d7dad83eeaaaf..5c5772cf5afe5 100644 --- a/llvm/utils/dtlto/mock.py +++ b/llvm/utils/dtlto/mock.py @@ -6,10 +6,10 @@ perform the backend compilation jobs. Usage: - python mock_distributor.py ... + python mock.py ... Arguments: - - , , ... : Input files to be copied. + - , , ... : Input files to be copied. - : JSON file describing the DTLTO jobs. The script performs the following: From a828b70095b78cba6d0957f94e0ab9a5d22b37fd Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Mon, 3 Mar 2025 13:23:51 +0000 Subject: [PATCH 16/22] Sync with https://github.com/llvm/llvm-project/pull/127749 - Pull in documentation improvements. --- llvm/docs/DTLTO.rst | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/llvm/docs/DTLTO.rst b/llvm/docs/DTLTO.rst index 3fed25a922f5a..0322e494099d2 100644 --- a/llvm/docs/DTLTO.rst +++ b/llvm/docs/DTLTO.rst @@ -91,10 +91,6 @@ JSON file provides the following information to the distributor: contribute. Distributors can use this to label build jobs for informational purposes. -- **Linker's version string**. - - Distributors can use this to determine if the invoked remote optimisation - tool is compatible. - - The list of **imports** required for each job. - The per-job list of bitcode files from which importing will occur. This is the same information that is emitted into import files for ThinLTO. @@ -175,8 +171,8 @@ Key Features of the Schema - **Command-Line Template**: Command-line options are stored in a common template to avoid duplication for each job. The template consists of an array of strings and arrays. The arrays are placeholders which reference per-job - paths. This allows the remote optimisation tool to be changed without updating - the distributors. + paths. This allows the remote compiler and its arguments to be changed without + updating the distributors. Command-Line Expansion Example ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -196,14 +192,14 @@ The example above generates the following backend compilation command for -o dtlto.1.51232.native.o -c -x ir dtlto.o \ -fthinlto-index=dtlto.1.51232.native.o.thinlto.bc --target=x86_64-sie-ps5 -This expansion scheme allows the remote optimization tool to be changed without -updating the distributors. For example, if the "args" field in the above example -was replaced with: +This expansion scheme allows the remote compiler to be changed without updating +the distributors. For example, if the "args" field in the above example was +replaced with: .. code-block:: json "args": [ - "custom-codgen-tool", + "custom-compiler", "-opt-level=2", "-profile-instrument-use-path=my.profdata", "-output", ["primary_output", 0], @@ -216,7 +212,7 @@ Then distributors can expand the command line without needing to be updated: .. code-block:: console - custom-codgen-tool -opt-level=2 -profile-instrument-use-path=my.profdata \ + custom-compiler -opt-level=2 -profile-instrument-use-path=my.profdata \ -output dtlto.1.51232.native.o -input dtlto.o \ -thinlto-index dtlto.1.51232.native.o.thinlto.bc -triple x86_64-sie-ps5 From a6e810695f53202d8703dbd2218957fa089d92cb Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 5 Mar 2025 14:59:46 +0000 Subject: [PATCH 17/22] > What is UID in this context? Should this be PID? I have changed this to be `PID`. Note that I expect the naming strategy to change in the future as we get feedback from more distribution systems. I have some concerns that mixing the PID into the filenames might prevent the caching systems for some distribution systems from working effectively. --- llvm/docs/DTLTO.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/docs/DTLTO.rst b/llvm/docs/DTLTO.rst index 0322e494099d2..d5f90b90a5737 100644 --- a/llvm/docs/DTLTO.rst +++ b/llvm/docs/DTLTO.rst @@ -112,15 +112,15 @@ include the stem of the bitcode module, or the output file that the LTO invocation is contributing to, to aid the user in identifying them: - **JSON Job Description File**: - - Format: `dtlto..dist-file.json` + - Format: `dtlto..dist-file.json` - Example: `dtlto.77380.dist-file.json` (for output file `dtlto.elf`). - **Object Files From Backend Compilations**: - - Format: `...native.o` + - Format: `...native.o` - Example: `my.1.77380.native.o` (for bitcode module `my.o`). - **Summary Index Shard Files**: - - Format: `...native.o.thinlto.bc` + - Format: `...native.o.thinlto.bc` - Example: `my.1.77380.native.o.thinlto.bc` (for bitcode module `my.o`). Temporary files are removed, by default, after the backend compilations complete. From 1eebf91464f05de10cd10f823a9903839e45019c Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 5 Mar 2025 15:08:41 +0000 Subject: [PATCH 18/22] > Maybe NOIMPORTFILES (they are imports files not index files)? Thanks. Fixed. > Redundant with earlier line Thanks. Remove the first check leaving only the -save-temps one. > Similar to above suggest s/INDEX/IMPORT/ Thanks. Fixed. --- llvm/test/ThinLTO/X86/dtlto/imports.test | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/llvm/test/ThinLTO/X86/dtlto/imports.test b/llvm/test/ThinLTO/X86/dtlto/imports.test index d03ac291fd2b7..b1f47ffb95ff9 100644 --- a/llvm/test/ThinLTO/X86/dtlto/imports.test +++ b/llvm/test/ThinLTO/X86/dtlto/imports.test @@ -38,22 +38,18 @@ INPUTS-NEXT: ] # This check ensures that we have failed for the expected reason. ERR: failed: DTLTO backend compilation: cannot open native object file: -# Check that imports files have not been created. -RUN: ls | FileCheck %s --check-prefix=NOINDEXFILES -NOINDEXFILES-NOT: imports - -# Check that imports files are not created with -save-temps. +# Check that imports files are not created even if -save-temps is active. RUN: not %{command} -save-temps 2>&1 \ RUN: | FileCheck %s --check-prefixes=ERR -RUN: ls | FileCheck %s --check-prefix=NOINDEXFILES -NOINDEXFILES-NOT: imports +RUN: ls | FileCheck %s --check-prefix=NOIMPORTFILES +NOIMPORTFILES-NOT: imports # Check that imports files are created with -thinlto-emit-imports. RUN: not %{command} -thinlto-emit-imports 2>&1 \ RUN: | FileCheck %s --check-prefixes=ERR -RUN: ls | FileCheck %s --check-prefix=INDEXFILES -INDEXFILES: 0.bc.imports -INDEXFILES: 1.bc.imports +RUN: ls | FileCheck %s --check-prefix=IMPORTFILES +IMPORTFILES: 0.bc.imports +IMPORTFILES: 1.bc.imports #--- 0.ll target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" From b46bfabbd2e1165f17429aa9c872512bcf24a1d8 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 5 Mar 2025 15:10:26 +0000 Subject: [PATCH 19/22] > Is this description correct? No. Thanks. Fixed now. --- llvm/lib/LTO/LTO.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 724bcac7a2771..3f0d6c5d64965 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -99,10 +99,9 @@ cl::list AdditionalThinLTODistributorArgs( "thinlto-distributor-arg", cl::desc("Additional arguments to pass to the ThinLTO distributor")); -cl::opt - ThinLTORemoteCompiler("thinlto-remote-compiler", - cl::desc("Additional arguments to pass to the " - "ThinLTO remote optimization tool")); +cl::opt ThinLTORemoteCompiler( + "thinlto-remote-compiler", + cl::desc("Compiler to invoke for the ThinLTO backend compilations")); cl::list ThinLTORemoteCompilerArgs("thinlto-remote-compiler-arg", From a98fe792ab13311ab458ee1900a99fad705b1450 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 5 Mar 2025 15:28:56 +0000 Subject: [PATCH 20/22] > This isn't really tracing the full "thin backend", should a different string be used here? Thanks. Nice comment. I have removed all the tracing code from the new backend now. I proposed to add time tracing in a separate later PR. Is that acceptable? --- llvm/lib/LTO/LTO.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 3f0d6c5d64965..a76521b4c8873 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -2281,9 +2281,6 @@ class OutOfProcessThinBackend : public CGThinBackend { assert(ModuleToDefinedGVSummaries.count(ModulePath)); BackendThreadPool.async( [=](Job &J, const FunctionImporter::ImportMapTy &ImportList) { - if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled) - timeTraceProfilerInitialize(Conf.TimeTraceGranularity, - "thin backend"); if (auto E = emitFiles(ImportList, J.ModuleID, J.SummaryIndexPath, J.ModuleID.str(), J.ImportFiles)) { std::unique_lock L(ErrMu); @@ -2292,8 +2289,6 @@ class OutOfProcessThinBackend : public CGThinBackend { else Err = std::move(E); } - if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled) - timeTraceProfilerFinishThread(); }, std::ref(J), std::ref(ImportList)); From 88e6151b3ca739a9f78d190987b70b7c794f1dd9 Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 5 Mar 2025 15:34:38 +0000 Subject: [PATCH 21/22] > What is IndexPath? Apologies! Something left over from an older revision. I have fixed it to read "(optionally) imports to disk". --- llvm/include/llvm/LTO/LTO.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index b7460b97a47e6..4d0eda30b8636 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -246,8 +246,8 @@ class ThinBackendProc { Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, StringRef ModulePath, const std::string &NewModulePath) const; - // Write sharded indices to SummaryPath, (optionally) imports - // IndexPath, and (optionally) record imports in ImportsFiles. + // Write sharded indices to SummaryPath, (optionally) imports to disk, and + // (optionally) record imports in ImportsFiles. Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, StringRef ModulePath, StringRef SummaryPath, const std::string &NewModulePath, From e87720819aa090c15a5f74326fecc0d60d61d73f Mon Sep 17 00:00:00 2001 From: Ben Dunbobbin Date: Wed, 5 Mar 2025 15:37:58 +0000 Subject: [PATCH 22/22] > Is the use of llvm-lto and opt here just to do testing before the clang and lld patches go in? If so, do we need cross project testing right now? Wondering if it would be better to just stick with the testing you have in this patch that is under llvm/test, and add the cross project testing using only external tools once those patches are in. This test checks that the DTLTO code that the code that creates Clang command lines generates the expected options and that those options are accepted by clang. I wrote it to use llvm-lto and opt so it could be part of this commit before the clang and lld patches go in. However, I think it is a nice principle to only use external tools in cross-project-tests. I have removed the test from this PR with the intention of adding it back later. Additionally if we can use your idea (https://github.com/llvm/llvm-project/pull/127749#discussion_r1976039748) of storing the Clang `-c` step options and applying them to the remote backend compilations then this test will be rewritten to test that behaviour instead. --- cross-project-tests/CMakeLists.txt | 13 +- cross-project-tests/dtlto/README.md | 3 - .../dtlto/dtlto-translate-options.ll | 142 ------------------ cross-project-tests/dtlto/lit.local.cfg | 2 - cross-project-tests/lit.cfg.py | 5 +- 5 files changed, 3 insertions(+), 162 deletions(-) delete mode 100644 cross-project-tests/dtlto/README.md delete mode 100644 cross-project-tests/dtlto/dtlto-translate-options.ll delete mode 100644 cross-project-tests/dtlto/lit.local.cfg diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt index c58119447ce0e..7f2fee48fda77 100644 --- a/cross-project-tests/CMakeLists.txt +++ b/cross-project-tests/CMakeLists.txt @@ -19,13 +19,11 @@ set(CROSS_PROJECT_TEST_DEPS FileCheck check-gdb-llvm-support count - llvm-config llvm-dwarfdump - llvm-lto2 + llvm-config llvm-objdump - not - opt split-file + not ) if ("clang" IN_LIST LLVM_ENABLE_PROJECTS) @@ -96,13 +94,6 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests" DEPENDS clang ) -# DTLTO tests. -add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests" - ${CMAKE_CURRENT_BINARY_DIR}/dtlto - EXCLUDE_FROM_CHECK_ALL - DEPENDS ${CROSS_PROJECT_TEST_DEPS} - ) - # Add check-cross-project-* targets. add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${CROSS_PROJECT_TEST_DEPS} diff --git a/cross-project-tests/dtlto/README.md b/cross-project-tests/dtlto/README.md deleted file mode 100644 index cfd9d3496ca42..0000000000000 --- a/cross-project-tests/dtlto/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Tests for DTLTO (integrated distributed ThinLTO) functionality. - -These are integration tests as DTLTO invokes `clang` for code-generation. \ No newline at end of file diff --git a/cross-project-tests/dtlto/dtlto-translate-options.ll b/cross-project-tests/dtlto/dtlto-translate-options.ll deleted file mode 100644 index 7f5888cf9e6c7..0000000000000 --- a/cross-project-tests/dtlto/dtlto-translate-options.ll +++ /dev/null @@ -1,142 +0,0 @@ -;; Check that the expected Clang arguments are generated by DTLTO for the -;; backend compilations and are accepted by Clang. - -; RUN: rm -rf %t && split-file %s %t && cd %t - -;; Generate bitcode files with a summary index. -; RUN: opt -thinlto-bc x86_64-unknown-linux-gnu.ll -o x86_64-unknown-linux-gnu.bc -; RUN: opt -thinlto-bc x86_64-pc-windows-msvc.ll -o x86_64-pc-windows-msvc.bc - - -;; Check that invalid arguments cause a Clang error. This property is relied on -;; by the actual testcases later in this test. -; RUN: not %clang -x ir x86_64-unknown-linux-gnu.ll \ -; RUN: -invalid-incorrect-not-an-option 2>&1 | FileCheck %s --check-prefix=SANITY1 -; SANITY1: unknown argument: '-invalid-incorrect-not-an-option' - - -;; Define a substitution used to simplify the testcases. -; DEFINE: %{distributor} = dummy -; DEFINE: %{extra_flags} = dummy -; DEFINE: %{triple} = dummy -; DEFINE: %{command} = llvm-lto2 run \ -; DEFINE: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/%{distributor} \ -; DEFINE: -thinlto-remote-compiler-arg=-Wunused-command-line-argument \ -; DEFINE: @%{triple}.rsp %{extra_flags} - - -;; Write common arguments to response files. - -; RUN: echo "x86_64-unknown-linux-gnu.bc -o x86_64-unknown-linux-gnu.o \ -; RUN: -dtlto \ -; RUN: -dtlto-distributor=%python \ -; RUN: -thinlto-remote-compiler=%clang \ -; RUN: -thinlto-remote-compiler-arg=-Werror \ -; RUN: -r=x86_64-unknown-linux-gnu.bc,globalfunc1,plx" > x86_64-unknown-linux-gnu.rsp - -; RUN: echo "x86_64-pc-windows-msvc.bc -o x86_64-pc-windows-msvc.o \ -; RUN: -dtlto \ -; RUN: -dtlto-distributor=%python \ -; RUN: -thinlto-remote-compiler=%clang \ -; RUN: -thinlto-remote-compiler-arg=-Werror \ -; RUN: -thinlto-remote-compiler-arg=-Wno-override-module \ -; RUN: -r=x86_64-pc-windows-msvc.bc,globalfunc2,plx" > x86_64-pc-windows-msvc.rsp - - -;; Check that boolean configuration states are translated as expected and Clang -;; accepts them. - -; RUN: echo " \ -; RUN: --addrsig=1 \ -; RUN: -function-sections=1 \ -; RUN: -data-sections=1" > on.rsp - -; RUN: echo " \ -; RUN: --addrsig=0 \ -; RUN: -function-sections=0 \ -; RUN: -data-sections=0" > off.rsp - -;; Perform DTLTO with configuration state set. -; REDEFINE: %{extra_flags} = @on.rsp -; REDEFINE: %{distributor} = local.py -; REDEFINE: %{triple} = x86_64-unknown-linux-gnu -; RUN: %{command} -; REDEFINE: %{distributor} = validate.py -; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=ON \ -; RUN: --implicit-check-not=-no-pgo-warn-mismatch -; ON-DAG: "-faddrsig" -; ON-DAG: "-ffunction-sections" -; ON-DAG: "-fdata-sections" - -;; Perform DTLTO with configuration state unset. -; REDEFINE: %{extra_flags} = @off.rsp -; REDEFINE: %{distributor} = local.py -; RUN: %{command} -; REDEFINE: %{distributor} = validate.py -; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=OFF -; OFF-NOT: --implicit-check-not=--faddrsig -; OFF-NOT: --implicit-check-not=--ffunction-sections -; OFF-NOT: --implicit-check-not=--fdata-sections -; OFF-NOT: --implicit-check-not=-no-pgo-warn-mismatch - - -;; Check optimization level. - -; RUN: llvm-lto2 run \ -; RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ -; RUN: @x86_64-unknown-linux-gnu.rsp \ -; RUN: -O3 - -; RUN: not llvm-lto2 run \ -; RUN: -thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \ -; RUN: @x86_64-unknown-linux-gnu.rsp \ -; RUN: -O3 2>&1 | FileCheck %s --check-prefix=OPTLEVEL -; OPTLEVEL-DAG: "-O3" - - -;; Check relocation model. - -; REDEFINE: %{extra_flags} = -relocation-model=pic -; REDEFINE: %{distributor} = local.py -; RUN: %{command} -; REDEFINE: %{distributor} = validate.py -; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=PIC -; PIC: -fpic - - -; REDEFINE: %{extra_flags} = -relocation-model=pic -; REDEFINE: %{distributor} = local.py -; REDEFINE: %{triple} = x86_64-pc-windows-msvc -; RUN: %{command} -; REDEFINE: %{distributor} = validate.py -; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=NOPIC -; REDEFINE: %{triple} = x86_64-unknown-linux-gnu -; NOPIC-NOT: -fpic - -;; Check specifying a sample profile. -; REDEFINE: %{extra_flags} = --lto-sample-profile-file="missing.profdata" -; REDEFINE: %{distributor} = local.py -; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE_ERR -; SAMPLE_PROFILE_ERR: no such file or directory: 'missing.profdata' -; REDEFINE: %{distributor} = validate.py -; RUN: not %{command} 2>&1 | FileCheck %s --check-prefix=SAMPLE_PROFILE -; SAMPLE_PROFILE-DAG: "-fprofile-sample-use=missing.profdata" - - -;--- x86_64-unknown-linux-gnu.ll -target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" -target triple = "x86_64-unknown-linux-gnu" - -define void @globalfunc1() { -entry: - ret void -} - -;--- x86_64-pc-windows-msvc.ll -target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" -target triple = "x86_64-pc-windows-msvc" - -define void @globalfunc2() { -entry: - ret void -} diff --git a/cross-project-tests/dtlto/lit.local.cfg b/cross-project-tests/dtlto/lit.local.cfg deleted file mode 100644 index 9a5f7aff60249..0000000000000 --- a/cross-project-tests/dtlto/lit.local.cfg +++ /dev/null @@ -1,2 +0,0 @@ -if any(feature not in config.available_features for feature in ["clang", "llvm-lto2", "opt"]): - config.unsupported = True diff --git a/cross-project-tests/lit.cfg.py b/cross-project-tests/lit.cfg.py index 3d27954c7545e..66fdd63632885 100644 --- a/cross-project-tests/lit.cfg.py +++ b/cross-project-tests/lit.cfg.py @@ -19,7 +19,7 @@ config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell) # suffixes: A list of file extensions to treat as test files. -config.suffixes = [".c", ".cl", ".cpp", ".m", ".ll"] +config.suffixes = [".c", ".cl", ".cpp", ".m"] # excludes: A list of directories to exclude from the testsuite. The 'Inputs' # subdirectories contain auxiliary inputs for various tests in their parent @@ -96,9 +96,6 @@ def get_required_attr(config, attr_name): if lldb_path is not None: config.available_features.add("lldb") -for tool in ["llvm-lto2", "opt"]: - if llvm_config.use_llvm_tool(tool): - config.available_features.add(tool) def configure_dexter_substitutions(): """Configure substitutions for host platform and return list of dependencies"""