Skip to content

Commit 9c9119a

Browse files
sam-mccallsam-mccall
committed
[clangd] Extend SymbolOrigin, stop serializing it
New values: - Split Dynamic into Open/Preamble - Add Background (previously was just Unknown) - Soon: stdlib index This requires extending to 16 bits, which fits within the padding of Symbol. Unfortunately we're also *serializing* SymbolOrigin as a fixed 8 bits. Stop serializing SymbolOrigin: - conceptually, the source is whoever indexes or *deserializes* a symbol - deserialization takes SymbolOrigin as a parameter and stamps it on each sym - this is a breaking format change Differential Revision: https://reviews.llvm.org/D115243
1 parent d8716cd commit 9c9119a

File tree

16 files changed

+54
-83
lines changed

16 files changed

+54
-83
lines changed

‎clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ class DiskBackedIndexStorage : public BackgroundIndexStorage {
5858
auto Buffer = llvm::MemoryBuffer::getFile(ShardPath);
5959
if (!Buffer)
6060
return nullptr;
61-
if (auto I = readIndexFile(Buffer->get()->getBuffer()))
61+
if (auto I =
62+
readIndexFile(Buffer->get()->getBuffer(), SymbolOrigin::Background))
6263
return std::make_unique<IndexFileIn>(std::move(*I));
6364
else
6465
elog("Error while reading shard {0}: {1}", ShardIdentifier,

‎clang-tools-extra/clangd/index/FileIndex.cpp

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,12 @@
2727
#include "clang/AST/ASTContext.h"
2828
#include "clang/Index/IndexingAction.h"
2929
#include "clang/Index/IndexingOptions.h"
30-
#include "clang/Lex/MacroInfo.h"
3130
#include "clang/Lex/Preprocessor.h"
3231
#include "llvm/ADT/DenseMap.h"
3332
#include "llvm/ADT/Optional.h"
3433
#include "llvm/ADT/STLExtras.h"
3534
#include "llvm/ADT/StringMap.h"
3635
#include "llvm/ADT/StringRef.h"
37-
#include "llvm/Support/Error.h"
3836
#include <algorithm>
3937
#include <memory>
4038
#include <tuple>
@@ -54,7 +52,8 @@ SlabTuple indexSymbols(ASTContext &AST, Preprocessor &PP,
5452
CollectorOpts.CollectIncludePath = true;
5553
CollectorOpts.Includes = &Includes;
5654
CollectorOpts.CountReferences = false;
57-
CollectorOpts.Origin = SymbolOrigin::Dynamic;
55+
CollectorOpts.Origin =
56+
IsIndexMainAST ? SymbolOrigin::Open : SymbolOrigin::Preamble;
5857
CollectorOpts.CollectMainFileRefs = CollectMainFileRefs;
5958
// We want stdlib implementation details in the index only if we've opened the
6059
// file in question. This does means xrefs won't work, though.

‎clang-tools-extra/clangd/index/Serialization.cpp

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,6 @@ void writeSymbol(const Symbol &Sym, const StringTableOut &Strings,
320320
writeLocation(Sym.CanonicalDeclaration, Strings, OS);
321321
writeVar(Sym.References, OS);
322322
OS.write(static_cast<uint8_t>(Sym.Flags));
323-
OS.write(static_cast<uint8_t>(Sym.Origin));
324323
writeVar(Strings.index(Sym.Signature), OS);
325324
writeVar(Strings.index(Sym.CompletionSnippetSuffix), OS);
326325
writeVar(Strings.index(Sym.Documentation), OS);
@@ -336,7 +335,8 @@ void writeSymbol(const Symbol &Sym, const StringTableOut &Strings,
336335
WriteInclude(Include);
337336
}
338337

339-
Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) {
338+
Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings,
339+
SymbolOrigin Origin) {
340340
Symbol Sym;
341341
Sym.ID = Data.consumeID();
342342
Sym.SymInfo.Kind = static_cast<index::SymbolKind>(Data.consume8());
@@ -348,7 +348,7 @@ Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) {
348348
Sym.CanonicalDeclaration = readLocation(Data, Strings);
349349
Sym.References = Data.consumeVar();
350350
Sym.Flags = static_cast<Symbol::SymbolFlag>(Data.consume8());
351-
Sym.Origin = static_cast<SymbolOrigin>(Data.consume8());
351+
Sym.Origin = Origin;
352352
Sym.Signature = Data.consumeString(Strings);
353353
Sym.CompletionSnippetSuffix = Data.consumeString(Strings);
354354
Sym.Documentation = Data.consumeString(Strings);
@@ -452,9 +452,10 @@ readCompileCommand(Reader CmdReader, llvm::ArrayRef<llvm::StringRef> Strings) {
452452
// The current versioning scheme is simple - non-current versions are rejected.
453453
// If you make a breaking change, bump this version number to invalidate stored
454454
// data. Later we may want to support some backward compatibility.
455-
constexpr static uint32_t Version = 16;
455+
constexpr static uint32_t Version = 17;
456456

457-
llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data) {
457+
llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data,
458+
SymbolOrigin Origin) {
458459
auto RIFF = riff::readFile(Data);
459460
if (!RIFF)
460461
return RIFF.takeError();
@@ -503,7 +504,7 @@ llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data) {
503504
Reader SymbolReader(Chunks.lookup("symb"));
504505
SymbolSlab::Builder Symbols;
505506
while (!SymbolReader.eof())
506-
Symbols.insert(readSymbol(SymbolReader, Strings->Strings));
507+
Symbols.insert(readSymbol(SymbolReader, Strings->Strings, Origin));
507508
if (SymbolReader.err())
508509
return error("malformed or truncated symbol");
509510
Result.Symbols = std::move(Symbols).build();
@@ -670,7 +671,7 @@ void writeRIFF(const IndexFileOut &Data, llvm::raw_ostream &OS) {
670671

671672
// Defined in YAMLSerialization.cpp.
672673
void writeYAML(const IndexFileOut &, llvm::raw_ostream &);
673-
llvm::Expected<IndexFileIn> readYAML(llvm::StringRef);
674+
llvm::Expected<IndexFileIn> readYAML(llvm::StringRef, SymbolOrigin Origin);
674675

675676
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) {
676677
switch (O.Format) {
@@ -684,11 +685,12 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) {
684685
return OS;
685686
}
686687

687-
llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef Data) {
688+
llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef Data,
689+
SymbolOrigin Origin) {
688690
if (Data.startswith("RIFF")) {
689-
return readRIFF(Data);
691+
return readRIFF(Data, Origin);
690692
}
691-
if (auto YAMLContents = readYAML(Data)) {
693+
if (auto YAMLContents = readYAML(Data, Origin)) {
692694
return std::move(*YAMLContents);
693695
} else {
694696
return error("Not a RIFF file and failed to parse as YAML: {0}",
@@ -697,7 +699,7 @@ llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef Data) {
697699
}
698700

699701
std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
700-
bool UseDex) {
702+
SymbolOrigin Origin, bool UseDex) {
701703
trace::Span OverallTracer("LoadIndex");
702704
auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
703705
if (!Buffer) {
@@ -710,7 +712,7 @@ std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef SymbolFilename,
710712
RelationSlab Relations;
711713
{
712714
trace::Span Tracer("ParseIndex");
713-
if (auto I = readIndexFile(Buffer->get()->getBuffer())) {
715+
if (auto I = readIndexFile(Buffer->get()->getBuffer(), Origin)) {
714716
if (I->Symbols)
715717
Symbols = std::move(*I->Symbols);
716718
if (I->Refs)

‎clang-tools-extra/clangd/index/Serialization.h

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ struct IndexFileIn {
4949
llvm::Optional<tooling::CompileCommand> Cmd;
5050
};
5151
// Parse an index file. The input must be a RIFF or YAML file.
52-
llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
52+
llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef, SymbolOrigin);
5353

5454
// Specifies the contents of an index file to be written.
5555
struct IndexFileOut {
@@ -79,16 +79,10 @@ std::string toYAML(const std::pair<SymbolID, ArrayRef<Ref>> &);
7979
std::string toYAML(const Relation &);
8080
std::string toYAML(const Ref &);
8181

82-
// Deserialize a single symbol from YAML.
83-
llvm::Expected<clangd::Symbol> symbolFromYAML(StringRef YAML,
84-
llvm::UniqueStringSaver *Strings);
85-
llvm::Expected<clangd::Ref> refFromYAML(StringRef YAML,
86-
llvm::UniqueStringSaver *Strings);
87-
8882
// Build an in-memory static index from an index file.
8983
// The size should be relatively small, so data can be managed in memory.
9084
std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef Filename,
91-
bool UseDex = true);
85+
SymbolOrigin Origin, bool UseDex = true);
9286

9387
} // namespace clangd
9488
} // namespace clang

‎clang-tools-extra/clangd/index/SymbolOrigin.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ namespace clangd {
1414
llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, SymbolOrigin O) {
1515
if (O == SymbolOrigin::Unknown)
1616
return OS << "unknown";
17-
constexpr static char Sigils[] = "ADSMIR67";
17+
constexpr static char Sigils[] = "AOSMIRP7B9012345";
1818
for (unsigned I = 0; I < sizeof(Sigils); ++I)
19-
if (static_cast<uint8_t>(O) & 1u << I)
19+
if (static_cast<uint16_t>(O) & 1u << I)
2020
OS << Sigils[I];
2121
return OS;
2222
}

‎clang-tools-extra/clangd/index/SymbolOrigin.h

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,27 +18,29 @@ namespace clangd {
1818
// Describes the source of information about a symbol.
1919
// Mainly useful for debugging, e.g. understanding code completion results.
2020
// This is a bitfield as information can be combined from several sources.
21-
enum class SymbolOrigin : uint8_t {
21+
enum class SymbolOrigin : uint16_t {
2222
Unknown = 0,
2323
AST = 1 << 0, // Directly from the AST (indexes should not set this).
24-
Dynamic = 1 << 1, // From the dynamic index of opened files.
25-
Static = 1 << 2, // From the static, externally-built index.
24+
Open = 1 << 1, // From the dynamic index of open files.
25+
Static = 1 << 2, // From a static, externally-built index.
2626
Merge = 1 << 3, // A non-trivial index merge was performed.
2727
Identifier = 1 << 4, // Raw identifiers in file.
2828
Remote = 1 << 5, // Remote index.
29-
// Remaining bits reserved for index implementations.
29+
Preamble = 1 << 6, // From the dynamic index of preambles.
30+
// 7 reserved
31+
Background = 1 << 8, // From the automatic project index.
3032
};
3133

3234
inline SymbolOrigin operator|(SymbolOrigin A, SymbolOrigin B) {
33-
return static_cast<SymbolOrigin>(static_cast<uint8_t>(A) |
34-
static_cast<uint8_t>(B));
35+
return static_cast<SymbolOrigin>(static_cast<uint16_t>(A) |
36+
static_cast<uint16_t>(B));
3537
}
3638
inline SymbolOrigin &operator|=(SymbolOrigin &A, SymbolOrigin B) {
3739
return A = A | B;
3840
}
3941
inline SymbolOrigin operator&(SymbolOrigin A, SymbolOrigin B) {
40-
return static_cast<SymbolOrigin>(static_cast<uint8_t>(A) &
41-
static_cast<uint8_t>(B));
42+
return static_cast<SymbolOrigin>(static_cast<uint16_t>(A) &
43+
static_cast<uint16_t>(B));
4244
}
4345

4446
llvm::raw_ostream &operator<<(llvm::raw_ostream &, SymbolOrigin);

‎clang-tools-extra/clangd/index/YAMLSerialization.cpp

Lines changed: 5 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@ using clang::clangd::RelationKind;
6969
using clang::clangd::Symbol;
7070
using clang::clangd::SymbolID;
7171
using clang::clangd::SymbolLocation;
72-
using clang::clangd::SymbolOrigin;
7372
using clang::index::SymbolInfo;
7473
using clang::index::SymbolKind;
7574
using clang::index::SymbolLanguage;
@@ -108,17 +107,6 @@ struct NormalizedSymbolFlag {
108107
uint8_t Flag = 0;
109108
};
110109

111-
struct NormalizedSymbolOrigin {
112-
NormalizedSymbolOrigin(IO &) {}
113-
NormalizedSymbolOrigin(IO &, SymbolOrigin O) {
114-
Origin = static_cast<uint8_t>(O);
115-
}
116-
117-
SymbolOrigin denormalize(IO &) { return static_cast<SymbolOrigin>(Origin); }
118-
119-
uint8_t Origin = 0;
120-
};
121-
122110
template <> struct MappingTraits<YPosition> {
123111
static void mapping(IO &IO, YPosition &Value) {
124112
IO.mapRequired("Line", Value.Line);
@@ -194,8 +182,6 @@ template <> struct MappingTraits<Symbol> {
194182
MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
195183
MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
196184
IO, Sym.Flags);
197-
MappingNormalization<NormalizedSymbolOrigin, SymbolOrigin> NSymbolOrigin(
198-
IO, Sym.Origin);
199185
IO.mapRequired("ID", NSymbolID->HexString);
200186
IO.mapRequired("Name", Sym.Name);
201187
IO.mapRequired("Scope", Sym.Scope);
@@ -204,7 +190,6 @@ template <> struct MappingTraits<Symbol> {
204190
SymbolLocation());
205191
IO.mapOptional("Definition", Sym.Definition, SymbolLocation());
206192
IO.mapOptional("References", Sym.References, 0u);
207-
IO.mapOptional("Origin", NSymbolOrigin->Origin);
208193
IO.mapOptional("Flags", NSymbolFlag->Flag);
209194
IO.mapOptional("Signature", Sym.Signature);
210195
IO.mapOptional("TemplateSpecializationArgs",
@@ -436,7 +421,8 @@ void writeYAML(const IndexFileOut &O, llvm::raw_ostream &OS) {
436421
}
437422
}
438423

439-
llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data) {
424+
llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data,
425+
SymbolOrigin Origin) {
440426
SymbolSlab::Builder Symbols;
441427
RefSlab::Builder Refs;
442428
RelationSlab::Builder Relations;
@@ -453,8 +439,10 @@ llvm::Expected<IndexFileIn> readYAML(llvm::StringRef Data) {
453439
if (Yin.error())
454440
return llvm::errorCodeToError(Yin.error());
455441

456-
if (Variant.Symbol)
442+
if (Variant.Symbol) {
443+
Variant.Symbol->Origin = Origin;
457444
Symbols.insert(*Variant.Symbol);
445+
}
458446
if (Variant.Refs)
459447
for (const auto &Ref : Variant.Refs->second)
460448
Refs.insert(Variant.Refs->first, Ref);
@@ -528,25 +516,5 @@ std::string toYAML(const Ref &R) {
528516
return Buf;
529517
}
530518

531-
llvm::Expected<clangd::Symbol>
532-
symbolFromYAML(StringRef YAML, llvm::UniqueStringSaver *Strings) {
533-
clangd::Symbol Deserialized;
534-
llvm::yaml::Input YAMLInput(YAML, Strings);
535-
if (YAMLInput.error())
536-
return error("Unable to deserialize Symbol from YAML: {0}", YAML);
537-
YAMLInput >> Deserialized;
538-
return Deserialized;
539-
}
540-
541-
llvm::Expected<clangd::Ref> refFromYAML(StringRef YAML,
542-
llvm::UniqueStringSaver *Strings) {
543-
clangd::Ref Deserialized;
544-
llvm::yaml::Input YAMLInput(YAML, Strings);
545-
if (YAMLInput.error())
546-
return error("Unable to deserialize Symbol from YAML: {0}", YAML);
547-
YAMLInput >> Deserialized;
548-
return Deserialized;
549-
}
550-
551519
} // namespace clangd
552520
} // namespace clang

‎clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,8 @@ class Export : public Command {
334334
}
335335

336336
// Auto-detects input format when parsing
337-
auto IndexIn = clang::clangd::readIndexFile(Buffer->get()->getBuffer());
337+
auto IndexIn = clang::clangd::readIndexFile(Buffer->get()->getBuffer(),
338+
SymbolOrigin::Static);
338339
if (!IndexIn) {
339340
llvm::errs() << llvm::toString(IndexIn.takeError()) << "\n";
340341
return;
@@ -374,7 +375,7 @@ std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
374375
return Index.startswith("remote:")
375376
? remote::getClient(Index.drop_front(strlen("remote:")),
376377
ProjectRoot)
377-
: loadIndex(Index, /*UseDex=*/true);
378+
: loadIndex(Index, SymbolOrigin::Static, /*UseDex=*/true);
378379
}
379380

380381
bool runCommand(std::string Request, const SymbolIndex &Index) {

‎clang-tools-extra/clangd/index/remote/Index.proto

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ message Symbol {
6666
optional string scope = 5;
6767
optional SymbolLocation canonical_declaration = 6;
6868
optional int32 references = 7;
69-
optional uint32 origin = 8;
69+
reserved 8;
7070
optional string signature = 9;
7171
optional string template_specialization_args = 10;
7272
optional string completion_snippet_suffix = 11;

‎clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,6 @@ llvm::Expected<Symbol> Marshaller::toProtobuf(const clangd::Symbol &From) {
269269
return Declaration.takeError();
270270
*Result.mutable_canonical_declaration() = *Declaration;
271271
Result.set_references(From.References);
272-
Result.set_origin(static_cast<uint32_t>(From.Origin));
273272
Result.set_signature(From.Signature.str());
274273
Result.set_template_specialization_args(
275274
From.TemplateSpecializationArgs.str());

0 commit comments

Comments
 (0)