[llvm-pdbutil] Move InputFile/FormatUtil/LinePrinter to PDB library.

At Sony we are developing llvm-dva

https://lists.llvm.org/pipermail/llvm-dev/2020-August/144174.html

For its PDB support, it requires functionality already present in
llvm-pdbutil.

We intend to move that functionaly into the PDB library to be
shared by both tools. That change will be done in 2 steps, that
will be submitted as 2 patches:

(1) Replace 'ExitOnError' with explicit error handling.
(2) Move the intended shared code to the PDB library.

Patch for step (1): https://reviews.llvm.org/D121801

This patch is for step (2).

Move InputFile.cpp[h], FormatUtil.cpp[h] and LinePrinter.cpp[h]
files to the debug PDB library.

It exposes the following functionality that can be used by tools:

- Open a PDB file.
- Get module debug stream.
- Traverse module sections.
- Traverse module subsections.

Most of the needed functionality is in InputFile, but there are
dependencies from LinePrinter and FormatUtil.

Some other functionality is in the following functions in
DumpOutputStyle.cpp file:

- iterateModuleSubsections
- getModuleDebugStream
- iterateOneModule
- iterateSymbolGroups
- iterateModuleSubsections

Only these specific functions from DumpOutputStyle are moved to
the PDB library.

Reviewed By: aganea, dblaikie, rnk

Differential Revision: https://reviews.llvm.org/D122226

Author: CarlosAlbertoEnciso

llvm/tools/llvm-pdbutil/FormatUtil.h renamed to llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h

@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TOOLS_LLVMPDBUTIL_FORMAT_UTIL_H
-#define LLVM_TOOLS_LLVMPDBUTIL_FORMAT_UTIL_H
+#ifndef LLVM_DEBUGINFO_PDB_NATIVE_FORMATUTIL_H
+#define LLVM_DEBUGINFO_PDB_NATIVE_FORMATUTIL_H
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
@@ -136,6 +136,6 @@ fmtle(support::detail::packed_endian_specific_integral<T, support::little,
           Value) {
   return detail::EndianAdapter<T>(std::move(Value));
 }
-}
+} // namespace pdb
 } // namespace llvm
 #endif

llvm/tools/llvm-pdbutil/InputFile.h renamed to llvm/include/llvm/DebugInfo/PDB/Native/InputFile.h

@@ -6,15 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TOOLS_LLVMPDBDUMP_INPUTFILE_H
-#define LLVM_TOOLS_LLVMPDBDUMP_INPUTFILE_H
+#ifndef LLVM_DEBUGINFO_PDB_NATIVE_INPUTFILE_H
+#define LLVM_DEBUGINFO_PDB_NATIVE_INPUTFILE_H
 
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h"
 #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h"
+#include "llvm/DebugInfo/PDB/Native/LinePrinter.h"
 #include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
@@ -54,6 +55,9 @@ class InputFile {
   getOrCreateTypeCollection(TypeCollectionKind Kind);
 
 public:
+  InputFile(PDBFile *Pdb) { PdbOrObj = Pdb; }
+  InputFile(object::COFFObjectFile *Obj) { PdbOrObj = Obj; }
+  InputFile(MemoryBuffer *Buffer) { PdbOrObj = Buffer; }
   ~InputFile();
   InputFile(InputFile &&Other) = default;
 
@@ -91,6 +95,7 @@ class SymbolGroup {
   explicit SymbolGroup(InputFile *File, uint32_t GroupIndex = 0);
 
   Expected<StringRef> getNameFromStringTable(uint32_t Offset) const;
+  Expected<StringRef> getNameFromChecksums(uint32_t Offset) const;
 
   void formatFromFileName(LinePrinter &Printer, StringRef File,
                           bool Append = false) const;
@@ -148,6 +153,80 @@ class SymbolGroupIterator
   SymbolGroup Value;
 };
 
+Expected<ModuleDebugStreamRef>
+getModuleDebugStream(PDBFile &File, StringRef &ModuleName, uint32_t Index);
+Expected<ModuleDebugStreamRef> getModuleDebugStream(PDBFile &File,
+                                                    uint32_t Index);
+
+bool shouldDumpSymbolGroup(uint32_t Idx, const SymbolGroup &Group);
+
+// TODO: Change these callbacks to be function_refs (de-templatify them).
+template <typename CallbackT>
+Error iterateOneModule(InputFile &File, const Optional<PrintScope> &HeaderScope,
+                       const SymbolGroup &SG, uint32_t Modi,
+                       CallbackT Callback) {
+  if (HeaderScope) {
+    HeaderScope->P.formatLine(
+        "Mod {0:4} | `{1}`: ",
+        fmt_align(Modi, AlignStyle::Right, HeaderScope->LabelWidth), SG.name());
+  }
+
+  AutoIndent Indent(HeaderScope);
+  return Callback(Modi, SG);
+}
+
+template <typename CallbackT>
+Error iterateSymbolGroups(InputFile &Input,
+                          const Optional<PrintScope> &HeaderScope,
+                          CallbackT Callback) {
+  AutoIndent Indent(HeaderScope);
+
+  if (llvm::pdb::Filters.DumpModi > 0) {
+    assert(llvm::pdb::Filters.DumpModi == 1);
+    uint32_t Modi = llvm::pdb::Filters.DumpModi;
+    SymbolGroup SG(&Input, Modi);
+    return iterateOneModule(Input, withLabelWidth(HeaderScope, NumDigits(Modi)),
+                            SG, Modi, Callback);
+  }
+
+  uint32_t I = 0;
+
+  for (const auto &SG : Input.symbol_groups()) {
+    if (shouldDumpSymbolGroup(I, SG))
+      if (auto Err =
+              iterateOneModule(Input, withLabelWidth(HeaderScope, NumDigits(I)),
+                               SG, I, Callback))
+        return Err;
+
+    ++I;
+  }
+  return Error::success();
+}
+
+template <typename SubsectionT>
+Error iterateModuleSubsections(
+    InputFile &File, const Optional<PrintScope> &HeaderScope,
+    llvm::function_ref<Error(uint32_t, const SymbolGroup &, SubsectionT &)>
+        Callback) {
+
+  return iterateSymbolGroups(
+      File, HeaderScope, [&](uint32_t Modi, const SymbolGroup &SG) -> Error {
+        for (const auto &SS : SG.getDebugSubsections()) {
+          SubsectionT Subsection;
+
+          if (SS.kind() != Subsection.kind())
+            continue;
+
+          BinaryStreamReader Reader(SS.getRecordData());
+          if (auto Err = Subsection.initialize(Reader))
+            continue;
+          if (auto Err = Callback(Modi, SG, Subsection))
+            return Err;
+        }
+        return Error::success();
+      });
+}
+
 } // namespace pdb
 } // namespace llvm
 

llvm/tools/llvm-pdbutil/LinePrinter.h renamed to llvm/include/llvm/DebugInfo/PDB/Native/LinePrinter.h

@@ -6,44 +6,62 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_TOOLS_LLVMPDBDUMP_LINEPRINTER_H
-#define LLVM_TOOLS_LLVMPDBDUMP_LINEPRINTER_H
+#ifndef LLVM_DEBUGINFO_PDB_NATIVE_LINEPRINTER_H
+#define LLVM_DEBUGINFO_PDB_NATIVE_LINEPRINTER_H
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/DebugInfo/PDB/Native/FormatUtil.h"
 #include "llvm/Support/BinaryStreamRef.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <list>
 
+// Container for filter options to control which elements will be printed.
+struct FilterOptions {
+  std::list<std::string> ExcludeTypes;
+  std::list<std::string> ExcludeSymbols;
+  std::list<std::string> ExcludeCompilands;
+  std::list<std::string> IncludeTypes;
+  std::list<std::string> IncludeSymbols;
+  std::list<std::string> IncludeCompilands;
+  uint32_t PaddingThreshold;
+  uint32_t SizeThreshold;
+  uint32_t DumpModi;
+  bool JustMyCode;
+};
+
 namespace llvm {
 namespace msf {
 class MSFStreamLayout;
 } // namespace msf
 namespace pdb {
 
+extern FilterOptions Filters;
+
 class ClassLayout;
 class PDBFile;
 
 class LinePrinter {
   friend class WithColor;
 
 public:
-  LinePrinter(int Indent, bool UseColor, raw_ostream &Stream);
+  LinePrinter(int Indent, bool UseColor, raw_ostream &Stream,
+              FilterOptions &Filters);
 
   void Indent(uint32_t Amount = 0);
   void Unindent(uint32_t Amount = 0);
   void NewLine();
 
   void printLine(const Twine &T);
   void print(const Twine &T);
-  template <typename... Ts> void formatLine(const char *Fmt, Ts &&... Items) {
+  template <typename... Ts> void formatLine(const char *Fmt, Ts &&...Items) {
     printLine(formatv(Fmt, std::forward<Ts>(Items)...));
   }
-  template <typename... Ts> void format(const char *Fmt, Ts &&... Items) {
+  template <typename... Ts> void format(const char *Fmt, Ts &&...Items) {
     print(formatv(Fmt, std::forward<Ts>(Items)...));
   }
 
@@ -161,7 +179,7 @@ class WithColor {
   raw_ostream &OS;
   bool UseColor;
 };
-}
-}
+} // namespace pdb
+} // namespace llvm
 
 #endif

llvm/lib/DebugInfo/PDB/CMakeLists.txt

@@ -47,12 +47,15 @@ add_pdb_impl_folder(Native
   Native/DbiStream.cpp
   Native/DbiStreamBuilder.cpp
   Native/EnumTables.cpp
+  Native/FormatUtil.cpp
   Native/GlobalsStream.cpp
   Native/Hash.cpp
   Native/HashTable.cpp
   Native/InfoStream.cpp
   Native/InfoStreamBuilder.cpp
   Native/InjectedSourceStream.cpp
+  Native/InputFile.cpp
+  Native/LinePrinter.cpp
   Native/ModuleDebugStream.cpp
   Native/NativeCompilandSymbol.cpp
   Native/NativeEnumGlobals.cpp

llvm/tools/llvm-pdbutil/FormatUtil.cpp renamed to llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp

@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "FormatUtil.h"
+#include "llvm/DebugInfo/PDB/Native/FormatUtil.h"
+
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/BinaryFormat/COFF.h"

llvm/tools/llvm-pdbutil/InputFile.cpp renamed to llvm/lib/DebugInfo/PDB/Native/InputFile.cpp

@@ -6,17 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "InputFile.h"
-
-#include "FormatUtil.h"
-#include "LinePrinter.h"
+#include "llvm/DebugInfo/PDB/Native/InputFile.h"
 
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h"
 #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Native/DbiStream.h"
+#include "llvm/DebugInfo/PDB/Native/FormatUtil.h"
+#include "llvm/DebugInfo/PDB/Native/LinePrinter.h"
 #include "llvm/DebugInfo/PDB/Native/NativeSession.h"
 #include "llvm/DebugInfo/PDB/Native/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Native/PDBStringTable.h"
@@ -35,8 +34,9 @@ using namespace llvm::pdb;
 InputFile::InputFile() {}
 InputFile::~InputFile() {}
 
-static Expected<ModuleDebugStreamRef>
-getModuleDebugStream(PDBFile &File, StringRef &ModuleName, uint32_t Index) {
+Expected<ModuleDebugStreamRef>
+llvm::pdb::getModuleDebugStream(PDBFile &File, StringRef &ModuleName,
+                                uint32_t Index) {
   Expected<DbiStream &> DbiOrErr = File.getPDBDbiStream();
   if (!DbiOrErr)
     return DbiOrErr.takeError();
@@ -65,6 +65,30 @@ getModuleDebugStream(PDBFile &File, StringRef &ModuleName, uint32_t Index) {
   return std::move(ModS);
 }
 
+Expected<ModuleDebugStreamRef> llvm::pdb::getModuleDebugStream(PDBFile &File,
+                                                               uint32_t Index) {
+  Expected<DbiStream &> DbiOrErr = File.getPDBDbiStream();
+  if (!DbiOrErr)
+    return DbiOrErr.takeError();
+  DbiStream &Dbi = *DbiOrErr;
+  const auto &Modules = Dbi.modules();
+  auto Modi = Modules.getModuleDescriptor(Index);
+
+  uint16_t ModiStream = Modi.getModuleStreamIndex();
+  if (ModiStream == kInvalidStreamIndex)
+    return make_error<RawError>(raw_error_code::no_stream,
+                                "Module stream not present");
+
+  auto ModStreamData = File.createIndexedStream(ModiStream);
+
+  ModuleDebugStreamRef ModS(Modi, std::move(ModStreamData));
+  if (Error Err = ModS.reload())
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Invalid module stream");
+
+  return std::move(ModS);
+}
+
 static inline bool isCodeViewDebugSubsection(object::SectionRef Section,
                                              StringRef Name,
                                              BinaryStreamReader &Reader) {
@@ -122,7 +146,7 @@ static std::string formatChecksumKind(FileChecksumKind Kind) {
 }
 
 template <typename... Args>
-static void formatInternal(LinePrinter &Printer, bool Append, Args &&... args) {
+static void formatInternal(LinePrinter &Printer, bool Append, Args &&...args) {
   if (Append)
     Printer.format(std::forward<Args>(args)...);
   else
@@ -211,6 +235,26 @@ Expected<StringRef> SymbolGroup::getNameFromStringTable(uint32_t Offset) const {
   return SC.strings().getString(Offset);
 }
 
+Expected<StringRef> SymbolGroup::getNameFromChecksums(uint32_t Offset) const {
+  StringRef Name;
+  if (!SC.hasChecksums()) {
+    return std::move(Name);
+  }
+
+  auto Iter = SC.checksums().getArray().at(Offset);
+  if (Iter == SC.checksums().getArray().end()) {
+    return std::move(Name);
+  }
+
+  uint32_t FO = Iter->FileNameOffset;
+  auto ExpectedFile = getNameFromStringTable(FO);
+  if (!ExpectedFile) {
+    return std::move(Name);
+  }
+
+  return *ExpectedFile;
+}
+
 void SymbolGroup::formatFromFileName(LinePrinter &Printer, StringRef File,
                                      bool Append) const {
   auto FC = ChecksumsByFile.find(File);
@@ -510,3 +554,33 @@ bool SymbolGroupIterator::isEnd() const {
   assert(SectionIter.hasValue());
   return *SectionIter == Value.File->obj().section_end();
 }
+
+static bool isMyCode(const SymbolGroup &Group) {
+  if (Group.getFile().isObj())
+    return true;
+
+  StringRef Name = Group.name();
+  if (Name.startswith("Import:"))
+    return false;
+  if (Name.endswith_insensitive(".dll"))
+    return false;
+  if (Name.equals_insensitive("* linker *"))
+    return false;
+  if (Name.startswith_insensitive("f:\\binaries\\Intermediate\\vctools"))
+    return false;
+  if (Name.startswith_insensitive("f:\\dd\\vctools\\crt"))
+    return false;
+  return true;
+}
+
+bool llvm::pdb::shouldDumpSymbolGroup(uint32_t Idx, const SymbolGroup &Group) {
+  if (llvm::pdb::Filters.JustMyCode && !isMyCode(Group))
+    return false;
+
+  // If the arg was not specified on the command line, always dump all modules.
+  if (llvm::pdb::Filters.DumpModi == 0)
+    return true;
+
+  // Otherwise, only dump if this is the same module specified.
+  return (llvm::pdb::Filters.DumpModi == Idx);
+}