feat: Add code nav for includes (#166)

Author: varungandhi-src

indexer/Indexer.cc

@@ -52,6 +52,15 @@ FileLocalSourceRange::fromNonEmpty(const clang::SourceManager &sourceManager,
           fileId};
 }
 
+// static
+FileLocalSourceRange
+FileLocalSourceRange::makeEmpty(const clang::SourceManager &sourceManager,
+                                clang::SourceLocation loc) {
+  auto presumedLoc = sourceManager.getPresumedLoc(loc);
+  return {presumedLoc.getLine(), presumedLoc.getColumn(), presumedLoc.getLine(),
+          presumedLoc.getColumn()};
+}
+
 void FileLocalSourceRange::addToOccurrence(scip::Occurrence &occ) const {
   occ.add_range(this->startLine - 1);
   occ.add_range(this->startColumn - 1);
@@ -195,6 +204,19 @@ void MacroIndexer::saveReference(
   this->saveOccurrence(refFileId, macroNameToken, defMacroInfo,
                        Role::Reference);
 }
+void MacroIndexer::saveInclude(clang::FileID containingFileId,
+                               clang::SourceRange pathRange,
+                               AbsolutePathRef includedFilePath) {
+  auto it = this->includeRanges.find({containingFileId});
+  if (it != this->includeRanges.end()) {
+    it->second->emplace_back(pathRange, includedFilePath);
+    return;
+  }
+  PerFileIncludeInfo vec{std::make_pair(pathRange, includedFilePath)};
+  this->includeRanges.insert(
+      {{containingFileId},
+       std::make_shared<PerFileIncludeInfo>(std::move(vec))});
+}
 
 void MacroIndexer::emitDocumentOccurrencesAndSymbols(
     bool deterministic, SymbolFormatter &symbolFormatter, clang::FileID fileId,
@@ -257,6 +279,19 @@ void MacroIndexer::emitExternalSymbols(bool deterministic,
           }));
 }
 
+void MacroIndexer::forEachIncludeInFile(
+    clang::FileID fileId,
+    absl::FunctionRef<void(clang::SourceRange, AbsolutePathRef)> callback)
+    const {
+  auto it = this->includeRanges.find({fileId});
+  if (it == this->includeRanges.end()) {
+    return;
+  }
+  for (auto &[range, path] : *it->second) {
+    callback(range, path);
+  }
+}
+
 TuIndexer::TuIndexer(const clang::SourceManager &sourceManager,
                      const clang::LangOptions &langOptions,
                      const clang::ASTContext &astContext,
@@ -267,6 +302,43 @@ TuIndexer::TuIndexer(const clang::SourceManager &sourceManager,
       getStableFileId(getStableFileId), externalSymbols(),
       forwardDeclarations() {}
 
+void TuIndexer::saveSyntheticFileDefinition(clang::FileID fileId,
+                                            StableFileId stableFileId) {
+  if (stableFileId.isSynthetic || !stableFileId.isInProject) {
+    return;
+  }
+  // Strictly speaking, the token at the start of the file could be any
+  // token, including a reference to a built-in type or a keyword. In
+  // such a situation, there would be no natural place to put the definition.
+  // We could use a zero-length occurrence, which would be more "correct"
+  // but less useful, since you couldn't trigger Find references.
+  auto fileStartLoc = this->sourceManager.getLocForStartOfFile(fileId);
+  auto symbol = this->symbolFormatter.getFileSymbol(stableFileId);
+  auto tokenLength = clang::Lexer::MeasureTokenLength(
+      fileStartLoc, this->sourceManager, this->langOptions);
+  if (tokenLength > 0) {
+    this->saveDefinition(symbol, fileStartLoc, scip::SymbolInformation{});
+    return;
+  }
+  auto range =
+      FileLocalSourceRange::makeEmpty(this->sourceManager, fileStartLoc);
+  this->saveOccurrenceImpl(symbol, range, fileId, scip::SymbolRole::Definition);
+}
+
+void TuIndexer::saveInclude(clang::SourceRange sourceRange,
+                            StableFileId stableFileId) {
+  if (stableFileId.isSynthetic) {
+    return;
+  }
+  auto symbol = this->symbolFormatter.getFileSymbol(stableFileId);
+  // #include can't come from macro expansions, so instead of having
+  // to write a generic saveReference method which needs to handle
+  // ranges in macro expansions, directly call saveOccurrenceImpl.
+  auto [range, fileId] =
+      FileLocalSourceRange::fromNonEmpty(this->sourceManager, sourceRange);
+  this->saveOccurrenceImpl(symbol, range, fileId, 0);
+}
+
 void TuIndexer::saveBindingDecl(const clang::BindingDecl &bindingDecl) {
   auto optSymbol = this->symbolFormatter.getBindingSymbol(bindingDecl);
   if (!optSymbol.has_value()) {
@@ -795,6 +867,13 @@ PartialDocument &TuIndexer::saveOccurrence(std::string_view symbol,
                                            clang::SourceLocation expansionLoc,
                                            int32_t allRoles) {
   auto [range, fileId] = this->getTokenExpansionRange(expansionLoc);
+  return this->saveOccurrenceImpl(symbol, range, fileId, allRoles);
+}
+
+PartialDocument &TuIndexer::saveOccurrenceImpl(std::string_view symbol,
+                                               FileLocalSourceRange range,
+                                               clang::FileID fileId,
+                                               int32_t allRoles) {
   scip::Occurrence occ;
   range.addToOccurrence(occ);
   occ.set_symbol(symbol.data(), symbol.size());

indexer/Indexer.h

@@ -5,10 +5,12 @@
 #include <cstdint>
 #include <string>
 #include <string_view>
+#include <utility>
 #include <vector>
 
 #include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
+#include "absl/functional/function_ref.h"
 
 #include "clang/AST/RawCommentList.h"
 #include "clang/Basic/SourceLocation.h"
@@ -86,6 +88,9 @@ struct FileLocalSourceRange {
   static std::pair<FileLocalSourceRange, clang::FileID>
   fromNonEmpty(const clang::SourceManager &, clang::SourceRange inclusiveRange);
 
+  static FileLocalSourceRange makeEmpty(const clang::SourceManager &,
+                                        clang::SourceLocation);
+
   void addToOccurrence(scip::Occurrence &occ) const;
 
   std::string debugToString() const;
@@ -160,9 +165,25 @@ class MacroIndexer final {
 
   absl::flat_hash_set<NonFileBasedMacro> nonFileBasedMacros;
 
+  using PerFileIncludeInfo =
+      std::vector<std::pair<clang::SourceRange, AbsolutePathRef>>;
+
+  /// Map storing #include information on a per-FileID basis.
+  ///
+  /// On seeing `#include "some/dir/header.h"` in A.cpp, the map
+  /// will have a key for A.cpp and the AbsolutePathRef in the value
+  /// corresponds to the absolute path for header.h.
+  /// (Ideally, we'd use a clang::FileID instead of an AbsolutePathRef,
+  /// but it's not clear if that's possible with the existing APIs
+  /// during #include processing).
+  absl::flat_hash_map<llvm_ext::AbslHashAdapter<clang::FileID>,
+                      // shared_ptr for copy-ability required by flat_hash_map
+                      std::shared_ptr<PerFileIncludeInfo>>
+      includeRanges;
+
 public:
   MacroIndexer(clang::SourceManager &m)
-      : sourceManager(&m), table(), nonFileBasedMacros() {}
+      : sourceManager(&m), table(), nonFileBasedMacros(), includeRanges() {}
   MacroIndexer(MacroIndexer &&) = default;
   MacroIndexer &operator=(MacroIndexer &&other) = default;
   MacroIndexer(const MacroIndexer &) = delete;
@@ -176,11 +197,19 @@ class MacroIndexer final {
   void saveDefinition(const clang::Token &macroNameToken,
                       const clang::MacroInfo *);
 
+  void saveInclude(clang::FileID fileContainingInclude,
+                   clang::SourceRange pathRange,
+                   AbsolutePathRef includedFilePath);
+
   void emitDocumentOccurrencesAndSymbols(bool deterministic, SymbolFormatter &,
                                          clang::FileID, scip::Document &);
   void emitExternalSymbols(bool deterministic, SymbolFormatter &,
                            scip::Index &);
 
+  void forEachIncludeInFile(
+      clang::FileID,
+      absl::FunctionRef<void(clang::SourceRange, AbsolutePathRef)>) const;
+
 private:
   /// Pre-condition: all arguments are valid/non-null.
   void saveOccurrence(clang::FileID occFileId, const clang::Token &macroToken,
@@ -224,6 +253,15 @@ class TuIndexer final {
   TuIndexer(const clang::SourceManager &, const clang::LangOptions &,
             const clang::ASTContext &, SymbolFormatter &, GetStableFileId);
 
+  /// Emit a fake 'definition' for a file, which can be used as a target
+  /// of Go to definition from #include, as well as the source for
+  // Find references to see where a header has been included.
+  void saveSyntheticFileDefinition(clang::FileID, StableFileId);
+
+  /// Emit a reference to the fake 'definition' for a file, allowing Go to
+  /// Definition from '#include'.
+  void saveInclude(clang::SourceRange, StableFileId);
+
   // See NOTE(ref: emit-vs-save) for naming conventions.
 #define SAVE_DECL(DeclName) \
   void save##DeclName##Decl(const clang::DeclName##Decl &);
@@ -289,6 +327,11 @@ class TuIndexer final {
                                   clang::SourceLocation loc,
                                   int32_t allRoles = 0);
 
+  PartialDocument &saveOccurrenceImpl(std::string_view symbol,
+                                      FileLocalSourceRange range,
+                                      clang::FileID fileId,
+                                      int32_t allRoles = 0);
+
   DocComment tryGetDocComment(const clang::Decl &) const;
 };
 

indexer/SymbolFormatter.cc

@@ -160,6 +160,27 @@ std::string_view SymbolFormatter::getMacroSymbol(clang::SourceLocation defLoc) {
   return std::string_view(newIt->second);
 }
 
+std::string_view SymbolFormatter::getFileSymbol(StableFileId stableFileId) {
+  auto it = this->fileSymbolCache.find(stableFileId);
+  if (it != this->fileSymbolCache.end()) {
+    return std::string_view(it->second);
+  }
+  auto name =
+      this->formatTemporary("<file>/{}", stableFileId.path.asStringView());
+  std::string out{};
+  SymbolBuilder{.packageName = "todo-pkg",
+                .packageVersion = "todo-version",
+                .descriptors = {DescriptorBuilder{
+                    .name = name, .suffix = scip::Descriptor::Namespace}}}
+      .formatTo(out);
+  auto [newIt, inserted] =
+      this->fileSymbolCache.emplace(stableFileId, std::move(out));
+  ENFORCE(
+      inserted,
+      "StableFileId key was missing earlier, so insert should've succeeded");
+  return std::string_view(newIt->second);
+}
+
 // NOTE(def: canonical-decl):
 // It is a little subtle as to why using getCanonicalDecl will
 // give correct results. In particular, the result of getCanonicalDecl

indexer/SymbolFormatter.h

@@ -12,9 +12,11 @@
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/SmallVector.h"
 
+#include "scip/scip.pb.h"
+
+#include "indexer/Derive.h"
 #include "indexer/LlvmAdapter.h"
 #include "indexer/Path.h"
-#include "scip/scip.pb.h"
 
 #define FOR_EACH_DECL_TO_BE_INDEXED(F) \
   F(Binding)                           \
@@ -112,6 +114,14 @@ struct StableFileId {
   bool isInProject;
   /// Track this for debugging.
   bool isSynthetic;
+
+  template <typename H>
+  friend H AbslHashValue(H h, const StableFileId &stableFileId) {
+    return H::combine(std::move(h), stableFileId.path, stableFileId.isInProject,
+                      stableFileId.isSynthetic);
+  }
+
+  DERIVE_EQ_ALL(StableFileId)
 };
 
 using GetStableFileId =
@@ -128,6 +138,7 @@ class SymbolFormatter final {
                       std::string>
       locationBasedCache;
   absl::flat_hash_map<const clang::Decl *, std::string> declBasedCache;
+  absl::flat_hash_map<StableFileId, std::string> fileSymbolCache;
   absl::flat_hash_map<llvm_ext::AbslHashAdapter<clang::FileID>, uint32_t>
       anonymousTypeCounters;
   absl::flat_hash_map<llvm_ext::AbslHashAdapter<clang::FileID>, uint32_t>
@@ -138,12 +149,15 @@ class SymbolFormatter final {
   SymbolFormatter(const clang::SourceManager &sourceManager,
                   GetStableFileId getStableFileId)
       : sourceManager(sourceManager), getStableFileId(getStableFileId),
-        locationBasedCache(), declBasedCache(), scratchBuffer() {}
+        locationBasedCache(), declBasedCache(), fileSymbolCache(),
+        localVariableCounters(), scratchBuffer() {}
   SymbolFormatter(const SymbolFormatter &) = delete;
   SymbolFormatter &operator=(const SymbolFormatter &) = delete;
 
   std::string_view getMacroSymbol(clang::SourceLocation defLoc);
 
+  std::string_view getFileSymbol(StableFileId);
+
 #define DECLARE_GET_SYMBOL(DeclName)                     \
   std::optional<std::string_view> get##DeclName##Symbol( \
       const clang::DeclName##Decl &);