Merge from 'main' to 'sycl-web' (88 commits)

  CONFLICT (content): Merge conflict in clang/lib/CodeGen/CodeGenAction.cpp

Author: smanna12

clang-tools-extra/clangd/Config.h

@@ -29,6 +29,8 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Regex.h"
+#include <functional>
 #include <string>
 #include <vector>
 
@@ -100,6 +102,12 @@ struct Config {
     } ClangTidy;
 
     UnusedIncludesPolicy UnusedIncludes = None;
+
+    /// IncludeCleaner will not diagnose usages of these headers matched by
+    /// these regexes.
+    struct {
+      std::vector<std::function<bool(llvm::StringRef)>> IgnoreHeader;
+    } Includes;
   } Diagnostics;
 
   /// Style of the codebase.

clang-tools-extra/clangd/ConfigCompile.cpp

@@ -45,7 +45,9 @@
 #include "llvm/Support/SMLoc.h"
 #include "llvm/Support/SourceMgr.h"
 #include <algorithm>
+#include <memory>
 #include <string>
+#include <vector>
 
 namespace clang {
 namespace clangd {
@@ -432,6 +434,7 @@ struct FragmentCompiler {
         Out.Apply.push_back([Val](const Params &, Config &C) {
           C.Diagnostics.UnusedIncludes = *Val;
         });
+    compile(std::move(F.Includes));
 
     compile(std::move(F.ClangTidy));
   }
@@ -507,6 +510,41 @@ struct FragmentCompiler {
     }
   }
 
+  void compile(Fragment::DiagnosticsBlock::IncludesBlock &&F) {
+#ifdef CLANGD_PATH_CASE_INSENSITIVE
+    static llvm::Regex::RegexFlags Flags = llvm::Regex::IgnoreCase;
+#else
+    static llvm::Regex::RegexFlags Flags = llvm::Regex::NoFlags;
+#endif
+    auto Filters = std::make_shared<std::vector<llvm::Regex>>();
+    for (auto &HeaderPattern : F.IgnoreHeader) {
+      // Anchor on the right.
+      std::string AnchoredPattern = "(" + *HeaderPattern + ")$";
+      llvm::Regex CompiledRegex(AnchoredPattern, Flags);
+      std::string RegexError;
+      if (!CompiledRegex.isValid(RegexError)) {
+        diag(Warning,
+             llvm::formatv("Invalid regular expression '{0}': {1}",
+                           *HeaderPattern, RegexError)
+                 .str(),
+             HeaderPattern.Range);
+        continue;
+      }
+      Filters->push_back(std::move(CompiledRegex));
+    }
+    if (Filters->empty())
+      return;
+    auto Filter = [Filters](llvm::StringRef Path) {
+      for (auto &Regex : *Filters)
+        if (Regex.match(Path))
+          return true;
+      return false;
+    };
+    Out.Apply.push_back([Filter](const Params &, Config &C) {
+      C.Diagnostics.Includes.IgnoreHeader.emplace_back(Filter);
+    });
+  }
+
   void compile(Fragment::CompletionBlock &&F) {
     if (F.AllScopes) {
       Out.Apply.push_back(

clang-tools-extra/clangd/ConfigFragment.h

@@ -232,6 +232,15 @@ struct Fragment {
     /// - None
     llvm::Optional<Located<std::string>> UnusedIncludes;
 
+    /// Controls IncludeCleaner diagnostics.
+    struct IncludesBlock {
+      /// Regexes that will be used to avoid diagnosing certain includes as
+      /// unused or missing. These can match any suffix of the header file in
+      /// question.
+      std::vector<Located<std::string>> IgnoreHeader;
+    };
+    IncludesBlock Includes;
+
     /// Controls how clang-tidy will run over the code base.
     ///
     /// The settings are merged with any settings found in .clang-tidy

clang-tools-extra/clangd/IncludeCleaner.cpp

@@ -23,10 +23,14 @@
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Tooling/Syntax/Tokens.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/Regex.h"
+#include <functional>
 
 namespace clang {
 namespace clangd {
@@ -233,7 +237,8 @@ void findReferencedMacros(const SourceManager &SM, Preprocessor &PP,
   }
 }
 
-static bool mayConsiderUnused(const Inclusion &Inc, ParsedAST &AST) {
+static bool mayConsiderUnused(const Inclusion &Inc, ParsedAST &AST,
+                              const Config &Cfg) {
   if (Inc.BehindPragmaKeep)
     return false;
 
@@ -258,6 +263,15 @@ static bool mayConsiderUnused(const Inclusion &Inc, ParsedAST &AST) {
          FE->getName());
     return false;
   }
+  for (auto &Filter : Cfg.Diagnostics.Includes.IgnoreHeader) {
+    // Convert the path to Unix slashes and try to match aginast the fiilter.
+    llvm::SmallString<64> Path(Inc.Resolved);
+    llvm::sys::path::native(Path, llvm::sys::path::Style::posix);
+    if (Filter(Inc.Resolved)) {
+      dlog("{0} header is filtered out by the configuration", FE->getName());
+      return false;
+    }
+  }
   return true;
 }
 
@@ -369,6 +383,7 @@ getUnused(ParsedAST &AST,
           const llvm::DenseSet<IncludeStructure::HeaderID> &ReferencedFiles,
           const llvm::StringSet<> &ReferencedPublicHeaders) {
   trace::Span Tracer("IncludeCleaner::getUnused");
+  const Config &Cfg = Config::current();
   std::vector<const Inclusion *> Unused;
   for (const Inclusion &MFI : AST.getIncludeStructure().MainFileIncludes) {
     if (!MFI.HeaderID)
@@ -377,7 +392,7 @@ getUnused(ParsedAST &AST,
       continue;
     auto IncludeID = static_cast<IncludeStructure::HeaderID>(*MFI.HeaderID);
     bool Used = ReferencedFiles.contains(IncludeID);
-    if (!Used && !mayConsiderUnused(MFI, AST)) {
+    if (!Used && !mayConsiderUnused(MFI, AST, Cfg)) {
       dlog("{0} was not used, but is not eligible to be diagnosed as unused",
            MFI.Written);
       continue;

clang-tools-extra/clangd/TUScheduler.cpp

@@ -798,7 +798,7 @@ ASTWorker::~ASTWorker() {
 
 void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags,
                        bool ContentChanged) {
-  std::string TaskName = llvm::formatv("Update ({0})", Inputs.Version);
+  llvm::StringLiteral TaskName = "Update";
   auto Task = [=]() mutable {
     // Get the actual command as `Inputs` does not have a command.
     // FIXME: some build systems like Bazel will take time to preparing

clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp

@@ -263,6 +263,24 @@ TEST_F(ConfigCompileTests, DiagnosticsIncludeCleaner) {
   EXPECT_TRUE(compileAndApply());
   EXPECT_EQ(Conf.Diagnostics.UnusedIncludes,
             Config::UnusedIncludesPolicy::Strict);
+
+  Frag = {};
+  EXPECT_TRUE(Conf.Diagnostics.Includes.IgnoreHeader.empty())
+      << Conf.Diagnostics.Includes.IgnoreHeader.size();
+  Frag.Diagnostics.Includes.IgnoreHeader.push_back(
+      Located<std::string>("foo.h"));
+  Frag.Diagnostics.Includes.IgnoreHeader.push_back(
+      Located<std::string>(".*inc"));
+  EXPECT_TRUE(compileAndApply());
+  auto HeaderFilter = [this](llvm::StringRef Path) {
+    for (auto &Filter : Conf.Diagnostics.Includes.IgnoreHeader) {
+      if (Filter(Path))
+        return true;
+    }
+    return false;
+  };
+  EXPECT_TRUE(HeaderFilter("foo.h"));
+  EXPECT_FALSE(HeaderFilter("bar.h"));
 }
 
 TEST_F(ConfigCompileTests, DiagnosticSuppression) {

clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp

@@ -1738,6 +1738,8 @@ TEST(DiagnosticsTest, IncludeCleaner) {
 ]]
   #include "used.h"
 
+  #include "ignore.h"
+
   #include <system_header.h>
 
   void foo() {
@@ -1754,12 +1756,19 @@ TEST(DiagnosticsTest, IncludeCleaner) {
     #pragma once
     void used() {}
   )cpp";
+  TU.AdditionalFiles["ignore.h"] = R"cpp(
+    #pragma once
+    void ignore() {}
+  )cpp";
   TU.AdditionalFiles["system/system_header.h"] = "";
   TU.ExtraArgs = {"-isystem" + testPath("system")};
   // Off by default.
   EXPECT_THAT(*TU.build().getDiagnostics(), IsEmpty());
   Config Cfg;
   Cfg.Diagnostics.UnusedIncludes = Config::UnusedIncludesPolicy::Strict;
+  // Set filtering.
+  Cfg.Diagnostics.Includes.IgnoreHeader.emplace_back(
+      [](llvm::StringRef Header) { return Header.endswith("ignore.h"); });
   WithContextValue WithCfg(Config::Key, std::move(Cfg));
   EXPECT_THAT(
       *TU.build().getDiagnostics(),

clang/docs/ClangCommandLineReference.rst

@@ -801,7 +801,7 @@ Generate Interface Stub Files, emit merged text not binary.
 
 Extract API information
 
-.. option:: -fopenmp-new-driver
+.. option:: -fopenmp-new-driver, fno-openmp-new-driver
 
 Use the new driver for OpenMP offloading.
 

clang/docs/MisExpect.rst

@@ -0,0 +1,75 @@
+===================
+Misexpect
+===================
+.. contents::
+
+.. toctree::
+   :maxdepth: 1
+
+When developers use ``llvm.expect`` intrinsics, i.e., through use of
+``__builtin_expect(...)``, they are trying to communicate how their code is
+expected to behave at runtime to the optimizer. These annotations, however, can
+be incorrect for a variety of reasons: changes to the code base invalidate them
+silently, the developer mis-annotated them (e.g., using ``LIKELY`` instead of
+``UNLIKELY``), or perhaps they assumed something incorrectly when they wrote
+the annotation. Regardless of why, it is useful to detect these situations so
+that the optimizer can make more useful decisions about the code. 
+
+MisExpect diagnostics are intended to help developers identify and address
+these situations, by comparing the branch weights added by the ``llvm.expect``
+intrinsic to those collected through profiling. Whenever these values are
+mismatched, a diagnostic is surfaced to the user. Details on how the checks
+operate in the LLVM backed can be found in LLVM's documentation.
+
+By default MisExpect checking is quite strict, because the use of the
+``llvm.expect`` intrinsic is designed for specialized cases, where the outcome
+of a condition is severely skewed. As a result, the optimizer can be extremely
+aggressive, which can result in performance degradation if the outcome is less
+predictable than the annotation suggests. Even when the annotation is correct
+90% of the time, it may be beneficial to either remove the annotation or to use
+a different intrinsic that can communicate the probability more directly.
+
+Because this may be too strict, MisExpect diagnostics are not enabled by
+default, and support an additional flag to tolerate some deviation from the
+exact thresholds. The ``-fdiagnostic-misexpect-tolerance=N`` accepts
+deviations when comparing branch weights within ``N%`` of the expected values.
+So passing ``-fdiagnostic-misexpect-tolerance=5`` will not report diagnostic messages
+if the branch weight from the profile is within 5% of the weight added by
+the ``llvm.expect`` intrinsic.
+
+MisExpect diagnostics are also available in the form of optimization remarks,
+which can be serialized and processed through the ``opt-viewer.py``
+scripts in LLVM.
+
+.. option:: -Rpass=misexpect
+
+  Enables optimization remarks for misexpect when profiling data conflicts with
+  use of ``llvm.expect`` intrinsics.
+
+
+.. option:: -Wmisexpect
+
+  Enables misexpect warnings when profiling data conflicts with use of
+  ``llvm.expect`` intrinsics.
+
+.. option:: -fdiagnostic-misexpect-tolerance=N
+
+   Relaxes misexpect checking to tolerate profiling values within N% of the 
+   expected branch weight. e.g., a value of ``N=5`` allows misexpect to check against
+   ``0.95 * Threshold``
+
+LLVM supports 4 types of profile formats: Frontend, IR, CS-IR, and
+Sampling. MisExpect Diagnostics are compatible with all Profiling formats.
+
++----------------+--------------------------------------------------------------------------------------+
+| Profile Type   | Description                                                                          |
++================+======================================================================================+
+| Frontend       | Profiling instrumentation added during compilation by the frontend, i.e. ``clang``   |
++----------------+--------------------------------------------------------------------------------------+
+| IR             | Profiling instrumentation added during by the LLVM backend                           |
++----------------+--------------------------------------------------------------------------------------+
+| CS-IR          | Context Sensitive IR based profiles                                                  |
++----------------+--------------------------------------------------------------------------------------+
+| Sampling       | Profiles collected through sampling with external tools, such as ``perf`` on Linux   |
++----------------+--------------------------------------------------------------------------------------+
+

clang/docs/OffloadingDesign.rst

@@ -17,11 +17,6 @@ application using Clang.
 OpenMP Offloading
 =================
 
-.. note::
-   This documentation describes Clang's behavior using the new offloading
-   driver. This currently must be enabled manually using
-   ``-fopenmp-new-driver``.
-
 Clang supports OpenMP target offloading to several different architectures such
 as NVPTX, AMDGPU, X86_64, Arm, and PowerPC. Offloading code is generated by
 Clang and then executed using the ``libomptarget`` runtime and the associated
@@ -226,15 +221,15 @@ A fat binary is a binary file that contains information intended for another
 device. We create a fat object by embedding the output of the device compilation
 stage into the host as a named section. The output from the device compilation
 is passed to the host backend using the ``-fembed-offload-object`` flag. This
-inserts the object as a global in the host's IR. The section name contains the
-target triple and architecture that the data corresponds to for later use.
-Typically we will also add an extra string to the section name to prevent it
-from being merged with other sections if the user performs relocatable linking
-on the object.
+embeds the device image into the ``.llvm.offloading`` section using a special
+binary format that behaves like a string map. This binary format is used to
+bundle metadata about the image so the linker can associate the proper device
+linking action with the image. Each device image will start with the magic bytes
+``0x10FF10AD``.
 
 .. code-block:: llvm
 
-  @llvm.embedded.object = private constant [1 x i8] c"\00", section ".llvm.offloading.nvptx64.sm_70."
+  @llvm.embedded.object = private constant [1 x i8] c"\00", section ".llvm.offloading"
 
 The device code will then be placed in the corresponding section one the backend
 is run on the host, creating a fat object. Using fat objects allows us to treat
@@ -250,7 +245,7 @@ will use this information when :ref:`Device Linking`.
     +==================================+====================================================================+
     | omp_offloading_entries           | Offloading entry information (see :ref:`table-tgt_offload_entry`)  |
     +----------------------------------+--------------------------------------------------------------------+
-    | .llvm.offloading.<triple>.<arch> | Embedded device object file for the target device and architecture |
+    | .llvm.offloading                 | Embedded device object file for the target device and architecture |
     +----------------------------------+--------------------------------------------------------------------+
 
 .. _Device Linking:
@@ -262,9 +257,10 @@ Objects containing :ref:`table-offloading_sections` require special handling to
 create an executable device image. This is done using a Clang tool, see
 :doc:`ClangLinkerWrapper` for more information. This tool works as a wrapper
 over the host linking job. It scans the input object files for the offloading
-sections and runs the appropriate device linking action. The linked device image
-is then :ref:`wrapped <Device Binary Wrapping>` to create the symbols used to load the
-device image and link it with the host.
+section ``.llvm.offloading``. The device files stored in this section are then
+extracted and passed tot he appropriate linking job. The linked device image is
+then :ref:`wrapped <Device Binary Wrapping>` to create the symbols used to load
+the device image and link it with the host.
 
 The linker wrapper tool supports linking bitcode files through link time
 optimization (LTO). This is used whenever the object files embedded in the host
@@ -438,19 +434,22 @@ This code is compiled using the following Clang flags.
 
     $ clang++ -fopenmp -fopenmp-targets=nvptx64 -O3 zaxpy.cpp -c
 
-The output section in the object file can be seen using the ``readelf`` utility
+The output section in the object file can be seen using the ``readelf`` utility.
+The ``.llvm.offloading`` section has the ``SHF_EXCLUDE`` flag so it will be
+removed from the final executable or shared library by the linker.
 
 .. code-block:: text
 
   $ llvm-readelf -WS zaxpy.o
-  [Nr] Name                                       Type
-  ...
-  [34] omp_offloading_entries                     PROGBITS
-  [35] .llvm.offloading.nvptx64-nvidia-cuda.sm_70 PROGBITS
+  Section Headers:
+  [Nr] Name                   Type     Address          Off    Size   ES Flg Lk Inf Al
+  [11] omp_offloading_entries PROGBITS 0000000000000000 0001f0 000040 00   A  0   0  1
+  [12] .llvm.offloading       PROGBITS 0000000000000000 000260 030950 00   E  0   0  8
+
 
 Compiling this file again will invoke the ``clang-linker-wrapper`` utility to
 extract and link the device code stored at the section named
-``.llvm.offloading.nvptx64-nvidia-cuda.sm_70`` and then use entries stored in
+``.llvm.offloading`` and then use entries stored in
 the section named ``omp_offloading_entries`` to create the symbols necessary for
 ``libomptarget`` to register the device image and call the entry function.