LLVM and SPIRV-LLVM-Translator pulldown (WW30)

LLVM: llvm/llvm-project@190518d
SPIRV-LLVM-Translator: KhronosGroup/SPIRV-LLVM-Translator@6904b38

Author: bb-sycl

bolt/include/bolt/Core/BinaryContext.h

@@ -39,6 +39,7 @@
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/raw_ostream.h"
 #include <functional>
+#include <list>
 #include <map>
 #include <set>
 #include <shared_mutex>
@@ -199,7 +200,7 @@ class BinaryContext {
   uint32_t DuplicatedJumpTables{0x10000000};
 
   /// Function fragments to skip.
-  std::vector<BinaryFunction *> FragmentsToSkip;
+  std::unordered_set<BinaryFunction *> FragmentsToSkip;
 
   /// The runtime library.
   std::unique_ptr<RuntimeLibrary> RtLibrary;
@@ -235,6 +236,18 @@ class BinaryContext {
     MIB = std::move(TargetBuilder);
   }
 
+  /// Return function fragments to skip.
+  const std::unordered_set<BinaryFunction *> &getFragmentsToSkip() {
+    return FragmentsToSkip;
+  }
+
+  /// Add function fragment to skip
+  void addFragmentsToSkip(BinaryFunction *Function) {
+    FragmentsToSkip.insert(Function);
+  }
+
+  void clearFragmentsToSkip() { FragmentsToSkip.clear(); }
+
   /// Given DWOId returns CU if it exists in DWOCUs.
   Optional<DWARFUnit *> getDWOCU(uint64_t DWOId);
 
@@ -475,15 +488,15 @@ class BinaryContext {
   /// If \p NextJTAddress is different from zero, it is used as an upper
   /// bound for jump table memory layout.
   ///
-  /// Optionally, populate \p Offsets with jump table entries. The entries
+  /// Optionally, populate \p Address from jump table entries. The entries
   /// could be partially populated if the jump table detection fails.
   bool analyzeJumpTable(const uint64_t Address,
                         const JumpTable::JumpTableType Type, BinaryFunction &BF,
                         const uint64_t NextJTAddress = 0,
-                        JumpTable::OffsetsType *Offsets = nullptr);
+                        JumpTable::AddressesType *EntriesAsAddress = nullptr);
 
   /// After jump table locations are established, this function will populate
-  /// their OffsetEntries based on memory contents.
+  /// their EntriesAsAddress based on memory contents.
   void populateJumpTables();
 
   /// Returns a jump table ID and label pointing to the duplicated jump table.
@@ -498,12 +511,12 @@ class BinaryContext {
   /// to function \p BF.
   std::string generateJumpTableName(const BinaryFunction &BF, uint64_t Address);
 
-  /// Free memory used by jump table offsets
-  void clearJumpTableOffsets() {
+  /// Free memory used by JumpTable's EntriesAsAddress
+  void clearJumpTableTempData() {
     for (auto &JTI : JumpTables) {
       JumpTable &JT = *JTI.second;
-      JumpTable::OffsetsType Temp;
-      Temp.swap(JT.OffsetEntries);
+      JumpTable::AddressesType Temp;
+      Temp.swap(JT.EntriesAsAddress);
     }
   }
   /// Return true if the array of bytes represents a valid code padding.
@@ -641,6 +654,10 @@ class BinaryContext {
   /// special linux kernel sections
   std::unordered_map<uint64_t, std::vector<LKInstructionMarkerInfo>> LKMarkers;
 
+  /// List of external addresses in the code that are not a function start
+  /// and are referenced from BinaryFunction.
+  std::list<std::pair<BinaryFunction *, uint64_t>> InterproceduralReferences;
+
   /// PseudoProbe decoder
   MCPseudoProbeDecoder ProbeDecoder;
 
@@ -884,8 +901,23 @@ class BinaryContext {
   bool registerFragment(BinaryFunction &TargetFunction,
                         BinaryFunction &Function) const;
 
-  /// Resolve inter-procedural dependencies from \p Function.
-  void processInterproceduralReferences(BinaryFunction &Function);
+  /// Add unterprocedural reference for \p Function to \p Address
+  void addInterproceduralReference(BinaryFunction *Function, uint64_t Address) {
+    InterproceduralReferences.push_back({Function, Address});
+  }
+
+  /// Used to fix the target of linker-generated AArch64 adrp + add
+  /// sequence with no relocation info.
+  void addAdrpAddRelocAArch64(BinaryFunction &BF, MCInst &LoadLowBits,
+                              MCInst &LoadHiBits, uint64_t Target);
+
+  /// Return true if AARch64 veneer was successfully matched at a given
+  /// \p Address and register veneer binary function if \p MatchOnly
+  /// argument is false.
+  bool handleAArch64Veneer(uint64_t Address, bool MatchOnly = false);
+
+  /// Resolve inter-procedural dependencies from
+  void processInterproceduralReferences();
 
   /// Skip functions with all parent and child fragments transitively.
   void skipMarkedFragments();

bolt/include/bolt/Core/BinaryFunction.h

@@ -253,10 +253,6 @@ class BinaryFunction {
 
   std::unique_ptr<BinaryLoopInfo> BLI;
 
-  /// Set of external addresses in the code that are not a function start
-  /// and are referenced from this function.
-  std::set<uint64_t> InterproceduralReferences;
-
   /// All labels in the function that are referenced via relocations from
   /// data objects. Typically these are jump table destinations and computed
   /// goto labels.
@@ -337,9 +333,9 @@ class BinaryFunction {
   /// True if the original entry point was patched.
   bool IsPatched{false};
 
-  /// True if the function contains jump table with entries pointing to
-  /// locations in fragments.
-  bool HasSplitJumpTable{false};
+  /// True if the function contains explicit or implicit indirect branch to its
+  /// split fragments, e.g., split jump table, landing pad in split fragment
+  bool HasIndirectTargetToSplitFragment{false};
 
   /// True if there are no control-flow edges with successors in other functions
   /// (i.e. if tail calls have edges to function-local basic blocks).
@@ -1161,8 +1157,8 @@ class BinaryFunction {
   /// Return the number of emitted instructions for this function.
   uint32_t getNumNonPseudos() const {
     uint32_t N = 0;
-    for (BinaryBasicBlock *const &BB : layout())
-      N += BB->getNumNonPseudos();
+    for (const BinaryBasicBlock &BB : blocks())
+      N += BB.getNumNonPseudos();
     return N;
   }
 
@@ -1441,9 +1437,12 @@ class BinaryFunction {
   /// otherwise processed.
   bool isPseudo() const { return IsPseudo; }
 
-  /// Return true if the function contains a jump table with entries pointing
-  /// to split fragments.
-  bool hasSplitJumpTable() const { return HasSplitJumpTable; }
+  /// Return true if the function contains explicit or implicit indirect branch
+  /// to its split fragments, e.g., split jump table, landing pad in split
+  /// fragment.
+  bool hasIndirectTargetToSplitFragment() const {
+    return HasIndirectTargetToSplitFragment;
+  }
 
   /// Return true if all CFG edges have local successors.
   bool hasCanonicalCFG() const { return HasCanonicalCFG; }
@@ -1838,7 +1837,9 @@ class BinaryFunction {
 
   void setIsPatched(bool V) { IsPatched = V; }
 
-  void setHasSplitJumpTable(bool V) { HasSplitJumpTable = V; }
+  void setHasIndirectTargetToSplitFragment(bool V) {
+    HasIndirectTargetToSplitFragment = V;
+  }
 
   void setHasCanonicalCFG(bool V) { HasCanonicalCFG = V; }
 
@@ -2334,13 +2335,13 @@ class BinaryFunction {
   size_t estimateHotSize(const bool UseSplitSize = true) const {
     size_t Estimate = 0;
     if (UseSplitSize && isSplit()) {
-      for (const BinaryBasicBlock *BB : BasicBlocksLayout)
-        if (!BB->isCold())
-          Estimate += BC.computeCodeSize(BB->begin(), BB->end());
+      for (const BinaryBasicBlock &BB : blocks())
+        if (!BB.isCold())
+          Estimate += BC.computeCodeSize(BB.begin(), BB.end());
     } else {
-      for (const BinaryBasicBlock *BB : BasicBlocksLayout)
-        if (BB->getKnownExecutionCount() != 0)
-          Estimate += BC.computeCodeSize(BB->begin(), BB->end());
+      for (const BinaryBasicBlock &BB : blocks())
+        if (BB.getKnownExecutionCount() != 0)
+          Estimate += BC.computeCodeSize(BB.begin(), BB.end());
     }
     return Estimate;
   }
@@ -2349,16 +2350,16 @@ class BinaryFunction {
     if (!isSplit())
       return estimateSize();
     size_t Estimate = 0;
-    for (const BinaryBasicBlock *BB : BasicBlocksLayout)
-      if (BB->isCold())
-        Estimate += BC.computeCodeSize(BB->begin(), BB->end());
+    for (const BinaryBasicBlock &BB : blocks())
+      if (BB.isCold())
+        Estimate += BC.computeCodeSize(BB.begin(), BB.end());
     return Estimate;
   }
 
   size_t estimateSize() const {
     size_t Estimate = 0;
-    for (const BinaryBasicBlock *BB : BasicBlocksLayout)
-      Estimate += BC.computeCodeSize(BB->begin(), BB->end());
+    for (const BinaryBasicBlock &BB : blocks())
+      Estimate += BC.computeCodeSize(BB.begin(), BB.end());
     return Estimate;
   }
 

bolt/include/bolt/Core/JumpTable.h

@@ -69,9 +69,9 @@ class JumpTable : public BinaryData {
   /// All the entries as labels.
   std::vector<MCSymbol *> Entries;
 
-  /// All the entries as offsets into a function. Invalid after CFG is built.
-  using OffsetsType = std::vector<uint64_t>;
-  OffsetsType OffsetEntries;
+  /// All the entries as absolute addresses. Invalid after disassembly is done.
+  using AddressesType = std::vector<uint64_t>;
+  AddressesType EntriesAsAddress;
 
   /// Map <Offset> -> <Label> used for embedded jump tables. Label at 0 offset
   /// is the main label for the jump table.
@@ -87,18 +87,17 @@ class JumpTable : public BinaryData {
   uint64_t Count{0};
 
   /// BinaryFunction this jump tables belongs to.
-  BinaryFunction *Parent{nullptr};
+  SmallVector<BinaryFunction *, 1> Parents;
 
 private:
   /// Constructor should only be called by a BinaryContext.
   JumpTable(MCSymbol &Symbol, uint64_t Address, size_t EntrySize,
-            JumpTableType Type, LabelMapType &&Labels, BinaryFunction &BF,
-            BinarySection &Section);
+            JumpTableType Type, LabelMapType &&Labels, BinarySection &Section);
 
 public:
   /// Return the size of the jump table.
   uint64_t getSize() const {
-    return std::max(OffsetEntries.size(), Entries.size()) * EntrySize;
+    return std::max(EntriesAsAddress.size(), Entries.size()) * EntrySize;
   }
 
   const MCSymbol *getFirstLabel() const {

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -1384,7 +1384,7 @@ class MCPlusBuilder {
     llvm_unreachable("not implemented");
   }
 
-  /// Return true if the instruction CurInst, in combination with the recent
+  /// Return not 0 if the instruction CurInst, in combination with the recent
   /// history of disassembled instructions supplied by [Begin, End), is a linker
   /// generated veneer/stub that needs patching. This happens in AArch64 when
   /// the code is large and the linker needs to generate stubs, but it does
@@ -1394,11 +1394,14 @@ class MCPlusBuilder {
   /// is put in TgtLowBits, and its pair in TgtHiBits. If the instruction in
   /// TgtHiBits does not have an immediate operand, but an expression, then
   /// this expression is put in TgtHiSym and Tgt only contains the lower bits.
-  virtual bool matchLinkerVeneer(InstructionIterator Begin,
-                                 InstructionIterator End, uint64_t Address,
-                                 const MCInst &CurInst, MCInst *&TargetHiBits,
-                                 MCInst *&TargetLowBits,
-                                 uint64_t &Target) const {
+  /// Return value is a total number of instructions that were used to create
+  /// a veneer.
+  virtual uint64_t matchLinkerVeneer(InstructionIterator Begin,
+                                     InstructionIterator End, uint64_t Address,
+                                     const MCInst &CurInst,
+                                     MCInst *&TargetHiBits,
+                                     MCInst *&TargetLowBits,
+                                     uint64_t &Target) const {
     llvm_unreachable("not implemented");
   }
 

bolt/include/bolt/Passes/AllocCombiner.h

@@ -17,6 +17,7 @@ namespace bolt {
 class AllocCombinerPass : public BinaryFunctionPass {
   /// Stats aggregating variables
   uint64_t NumCombined{0};
+  uint64_t DynamicCountCombined{0};
   DenseSet<const BinaryFunction *> FuncsChanged;
 
   void combineAdjustments(BinaryFunction &BF);

bolt/include/bolt/Passes/DataflowAnalysis.h

@@ -315,6 +315,8 @@ class DataflowAnalysis {
   void run() {
     derived().preflight();
 
+    if (Func.begin() == Func.end())
+      return;
     // Initialize state for all points of the function
     for (BinaryBasicBlock &BB : Func) {
       StateTy &St = getOrCreateStateAt(BB);
@@ -324,7 +326,6 @@ class DataflowAnalysis {
         St = derived().getStartingStateAtPoint(Inst);
       }
     }
-    assert(Func.begin() != Func.end() && "Unexpected empty function");
 
     std::queue<BinaryBasicBlock *> Worklist;
     // TODO: Pushing this in a DFS ordering will greatly speed up the dataflow

bolt/include/bolt/Passes/FrameAnalysis.h

@@ -121,6 +121,10 @@ class FrameAnalysis {
   /// Set of functions that require the stack to be 16B aligned
   DenseSet<const BinaryFunction *> FunctionsRequireAlignment;
 
+  /// Set of functions that performs computations with stack addresses and
+  /// complicates our understanding of aliasing of stack spaces.
+  DenseSet<const BinaryFunction *> FunctionsWithStackArithmetic;
+
   /// Owns ArgAccesses for all instructions. References to elements are
   /// attached to instructions as indexes to this vector, in MCAnnotations.
   std::vector<ArgAccesses> ArgAccessesVector;
@@ -130,7 +134,6 @@ class FrameAnalysis {
   /// Analysis stats counters
   uint64_t NumFunctionsNotOptimized{0};
   uint64_t NumFunctionsFailedRestoreFI{0};
-  uint64_t CountFunctionsNotOptimized{0};
   uint64_t CountFunctionsFailedRestoreFI{0};
   uint64_t CountDenominator{0};
 
@@ -184,6 +187,12 @@ class FrameAnalysis {
     return FunctionsRequireAlignment.count(&Func);
   }
 
+  /// Return true if \p Func does computation with the address of any stack
+  /// position, meaning we have limited alias analysis on this function.
+  bool hasStackArithmetic(const BinaryFunction &Func) const {
+    return FunctionsWithStackArithmetic.count(&Func);
+  }
+
   /// Functions for retrieving our specific MCAnnotation data from instructions
   ErrorOr<ArgAccesses &> getArgAccessesFor(const MCInst &Inst);
 

bolt/include/bolt/Passes/FrameOptimizer.h

@@ -77,9 +77,12 @@ class FrameOptimizerPass : public BinaryFunctionPass {
   /// Stats aggregating variables
   uint64_t NumRedundantLoads{0};
   uint64_t NumRedundantStores{0};
-  uint64_t NumLoadsChangedToReg{0};
-  uint64_t NumLoadsChangedToImm{0};
+  uint64_t FreqRedundantLoads{0};
+  uint64_t FreqRedundantStores{0};
+  uint64_t FreqLoadsChangedToReg{0};
+  uint64_t FreqLoadsChangedToImm{0};
   uint64_t NumLoadsDeleted{0};
+  uint64_t FreqLoadsDeleted{0};
 
   DenseSet<const BinaryFunction *> FuncsChanged;
 

bolt/include/bolt/Passes/MCF.h

@@ -13,6 +13,7 @@ namespace llvm {
 namespace bolt {
 
 class BinaryFunction;
+class DataflowInfoManager;
 
 enum MCFCostFunction : char {
   MCF_DISABLE = 0,
@@ -22,6 +23,12 @@ enum MCFCostFunction : char {
   MCF_BLAMEFTS
 };
 
+/// Implement the idea in "SamplePGO - The Power of Profile Guided Optimizations
+/// without the Usability Burden" by Diego Novillo to make basic block counts
+/// equal if we show that A dominates B, B post-dominates A and they are in the
+/// same loop and same loop nesting level.
+void equalizeBBCounts(DataflowInfoManager &Info, BinaryFunction &BF);
+
 /// Fill edge counts based on the basic block count. Used in nonLBR mode when
 /// we only have bb count.
 void estimateEdgeCounts(BinaryFunction &BF);

bolt/include/bolt/Passes/ShrinkWrapping.h

@@ -303,13 +303,16 @@ class ShrinkWrapping {
   std::vector<int64_t> PopOffsetByReg;
   std::vector<MCPhysReg> DomOrder;
   CalleeSavedAnalysis CSA;
-  std::vector<SmallSetVector<MCInst *, 4>> SavePos;
-  std::vector<uint64_t> BestSaveCount;
-  std::vector<MCInst *> BestSavePos;
+  std::vector<std::vector<uint64_t>> BestSaveCount;
+  std::vector<std::vector<MCInst *>> BestSavePos;
 
   /// Pass stats
   static std::atomic_uint64_t SpillsMovedRegularMode;
   static std::atomic_uint64_t SpillsMovedPushPopMode;
+  static std::atomic_uint64_t SpillsMovedDynamicCount;
+  static std::atomic_uint64_t SpillsFailedDynamicCount;
+  static std::atomic_uint64_t InstrDynamicCount;
+  static std::atomic_uint64_t StoreDynamicCount;
 
   Optional<unsigned> AnnotationIndex;
 
@@ -515,7 +518,7 @@ class ShrinkWrapping {
         BC.MIB->removeAnnotation(Inst, getAnnotationIndex());
   }
 
-  bool perform();
+  bool perform(bool HotOnly = false);
 
   static void printStats();
 };