Skip to content

Commit d4b7c0d

Browse files
tobias-stadlertobias-stadler
authored
[Remarks] Auto-detect remark parser format (#144554)
Add remark format 'Auto', which performs automatic detection of the remark format using the magic numbers at the beginning of the remarks files. The RemarkLinker already did something similar, so we streamlined this and exposed this to llvm-remarkutil.
1 parent 67c52aa commit d4b7c0d

File tree

15 files changed

+76
-33
lines changed

15 files changed

+76
-33
lines changed

‎llvm/include/llvm/Remarks/RemarkFormat.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,17 @@ namespace remarks {
2323
constexpr StringLiteral Magic("REMARKS");
2424

2525
/// The format used for serializing/deserializing remarks.
26-
enum class Format { Unknown, YAML, Bitstream };
26+
enum class Format { Unknown, Auto, YAML, Bitstream };
2727

2828
/// Parse and validate a string for the remark format.
2929
LLVM_ABI Expected<Format> parseFormat(StringRef FormatStr);
3030

3131
/// Parse and validate a magic number to a remark format.
3232
LLVM_ABI Expected<Format> magicToFormat(StringRef Magic);
3333

34+
/// Detect format based on selected format and magic number
35+
LLVM_ABI Expected<Format> detectFormat(Format Selected, StringRef Magic);
36+
3437
} // end namespace remarks
3538
} // end namespace llvm
3639

‎llvm/include/llvm/Remarks/RemarkLinker.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,13 +80,12 @@ struct RemarkLinker {
8080
/// \p Buffer.
8181
/// \p Buffer can be either a standalone remark container or just
8282
/// metadata. This takes care of uniquing and merging the remarks.
83-
LLVM_ABI Error link(StringRef Buffer,
84-
std::optional<Format> RemarkFormat = std::nullopt);
83+
LLVM_ABI Error link(StringRef Buffer, Format RemarkFormat = Format::Auto);
8584

8685
/// Link the remarks found in \p Obj by looking for the right section and
8786
/// calling the method above.
8887
LLVM_ABI Error link(const object::ObjectFile &Obj,
89-
std::optional<Format> RemarkFormat = std::nullopt);
88+
Format RemarkFormat = Format::Auto);
9089

9190
/// Serialize the linked remarks to the stream \p OS, using the format \p
9291
/// RemarkFormat.

‎llvm/lib/Remarks/RemarkFormat.cpp

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,22 @@ Expected<Format> llvm::remarks::magicToFormat(StringRef MagicStr) {
4242

4343
if (Result == Format::Unknown)
4444
return createStringError(std::make_error_code(std::errc::invalid_argument),
45-
"Unknown remark magic: '%s'", MagicStr.data());
45+
"Automatic detection of remark format failed. "
46+
"Unknown magic number: '%.4s'",
47+
MagicStr.data());
4648
return Result;
4749
}
50+
51+
Expected<Format> llvm::remarks::detectFormat(Format Selected,
52+
StringRef MagicStr) {
53+
if (Selected == Format::Unknown)
54+
return createStringError(std::make_error_code(std::errc::invalid_argument),
55+
"Unknown remark parser format.");
56+
if (Selected != Format::Auto)
57+
return Selected;
58+
59+
// Empty files are valid bitstream files
60+
if (MagicStr.empty())
61+
return Format::Bitstream;
62+
return magicToFormat(MagicStr);
63+
}

‎llvm/lib/Remarks/RemarkLinker.cpp

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -66,17 +66,10 @@ void RemarkLinker::setExternalFilePrependPath(StringRef PrependPathIn) {
6666
PrependPath = std::string(PrependPathIn);
6767
}
6868

69-
Error RemarkLinker::link(StringRef Buffer, std::optional<Format> RemarkFormat) {
70-
if (!RemarkFormat) {
71-
Expected<Format> ParserFormat = magicToFormat(Buffer);
72-
if (!ParserFormat)
73-
return ParserFormat.takeError();
74-
RemarkFormat = *ParserFormat;
75-
}
76-
69+
Error RemarkLinker::link(StringRef Buffer, Format RemarkFormat) {
7770
Expected<std::unique_ptr<RemarkParser>> MaybeParser =
7871
createRemarkParserFromMeta(
79-
*RemarkFormat, Buffer,
72+
RemarkFormat, Buffer,
8073
PrependPath ? std::optional<StringRef>(StringRef(*PrependPath))
8174
: std::optional<StringRef>());
8275
if (!MaybeParser)
@@ -102,8 +95,7 @@ Error RemarkLinker::link(StringRef Buffer, std::optional<Format> RemarkFormat) {
10295
return Error::success();
10396
}
10497

105-
Error RemarkLinker::link(const object::ObjectFile &Obj,
106-
std::optional<Format> RemarkFormat) {
98+
Error RemarkLinker::link(const object::ObjectFile &Obj, Format RemarkFormat) {
10799
Expected<std::optional<StringRef>> SectionOrErr =
108100
getRemarksSectionContents(Obj);
109101
if (!SectionOrErr)

‎llvm/lib/Remarks/RemarkParser.cpp

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "BitstreamRemarkParser.h"
1616
#include "YAMLRemarkParser.h"
1717
#include "llvm-c/Remarks.h"
18+
#include "llvm/Remarks/RemarkFormat.h"
1819
#include "llvm/Support/CBindingWrapping.h"
1920
#include <optional>
2021

@@ -50,14 +51,18 @@ Expected<StringRef> ParsedStringTable::operator[](size_t Index) const {
5051

5152
Expected<std::unique_ptr<RemarkParser>>
5253
llvm::remarks::createRemarkParser(Format ParserFormat, StringRef Buf) {
53-
switch (ParserFormat) {
54+
auto DetectedFormat = detectFormat(ParserFormat, Buf);
55+
if (!DetectedFormat)
56+
return DetectedFormat.takeError();
57+
58+
switch (*DetectedFormat) {
5459
case Format::YAML:
5560
return std::make_unique<YAMLRemarkParser>(Buf);
5661
case Format::Bitstream:
5762
return std::make_unique<BitstreamRemarkParser>(Buf);
5863
case Format::Unknown:
59-
return createStringError(std::make_error_code(std::errc::invalid_argument),
60-
"Unknown remark parser format.");
64+
case Format::Auto:
65+
break;
6166
}
6267
llvm_unreachable("unhandled ParseFormat");
6368
}
@@ -66,15 +71,19 @@ Expected<std::unique_ptr<RemarkParser>>
6671
llvm::remarks::createRemarkParserFromMeta(
6772
Format ParserFormat, StringRef Buf,
6873
std::optional<StringRef> ExternalFilePrependPath) {
69-
switch (ParserFormat) {
74+
auto DetectedFormat = detectFormat(ParserFormat, Buf);
75+
if (!DetectedFormat)
76+
return DetectedFormat.takeError();
77+
78+
switch (*DetectedFormat) {
7079
case Format::YAML:
7180
return createYAMLParserFromMeta(Buf, std::move(ExternalFilePrependPath));
7281
case Format::Bitstream:
7382
return createBitstreamParserFromMeta(Buf,
7483
std::move(ExternalFilePrependPath));
7584
case Format::Unknown:
76-
return createStringError(std::make_error_code(std::errc::invalid_argument),
77-
"Unknown remark parser format.");
85+
case Format::Auto:
86+
break;
7887
}
7988
llvm_unreachable("unhandled ParseFormat");
8089
}

‎llvm/lib/Remarks/RemarkSerializer.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
2222
raw_ostream &OS) {
2323
switch (RemarksFormat) {
2424
case Format::Unknown:
25+
case Format::Auto:
2526
return createStringError(std::errc::invalid_argument,
26-
"Unknown remark serializer format.");
27+
"Invalid remark serializer format.");
2728
case Format::YAML:
2829
return std::make_unique<YAMLRemarkSerializer>(OS, Mode);
2930
case Format::Bitstream:
@@ -37,8 +38,9 @@ remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
3738
raw_ostream &OS, remarks::StringTable StrTab) {
3839
switch (RemarksFormat) {
3940
case Format::Unknown:
41+
case Format::Auto:
4042
return createStringError(std::errc::invalid_argument,
41-
"Unknown remark serializer format.");
43+
"Invalid remark serializer format.");
4244
case Format::YAML:
4345
return std::make_unique<YAMLRemarkSerializer>(OS, Mode, std::move(StrTab));
4446
case Format::Bitstream:

‎llvm/test/tools/llvm-remarkutil/Inputs/broken-remark-magic.bitstream

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
12345678

‎llvm/test/tools/llvm-remarkutil/annotation-count.test

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
RUN: llvm-remarkutil annotation-count --parser=yaml --annotation-type=remark %p/Inputs/annotation-count.yaml | FileCheck %s
2+
RUN: llvm-remarkutil annotation-count --annotation-type=remark %p/Inputs/annotation-count.yaml | FileCheck %s
23
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remark | FileCheck %s
4+
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil annotation-count --annotation-type=remark | FileCheck %s
35
RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function --remark-name="AnnotationSummary" %p/Inputs/annotation-count.yaml | FileCheck %s --check-prefix=COUNT-CHECK
46
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil count --parser=bitstream --count-by=arg --group-by=function --remark-name="AnnotationSummary" | FileCheck %s --check-prefix=COUNT-CHECK
57

‎llvm/test/tools/llvm-remarkutil/broken-bitstream-remark-magic.test

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
RUN: not llvm-remarkutil instruction-count %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
2+
RUN: not llvm-remarkutil instruction-mix %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
3+
RUN: not llvm-remarkutil annotation-count --annotation-type=remark %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
4+
RUN: not llvm-remarkutil count %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
5+
6+
CHECK: error: Automatic detection of remark format failed. Unknown magic number: '1234'

‎llvm/test/tools/llvm-remarkutil/empty-file.test

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ RUN: llvm-remarkutil instruction-count --parser=bitstream %p/Inputs/empty-file -
88
RUN: llvm-remarkutil instruction-mix --parser=bitstream %p/Inputs/empty-file --report_style=csv -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=MIXBITSTREAM
99
RUN: llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remark %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=ANNOTATIONBITSTREAM
1010
RUN: llvm-remarkutil count --parser=bitstream %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=COUNTBITSTREAM
11+
; Parser format auto-detection should treat empty files as bitstream files
12+
RUN: llvm-remarkutil instruction-count %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=SIZEBITSTREAM
13+
RUN: llvm-remarkutil instruction-mix %p/Inputs/empty-file --report_style=csv -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=MIXBITSTREAM
14+
RUN: llvm-remarkutil annotation-count --annotation-type=remark %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=ANNOTATIONBITSTREAM
15+
RUN: llvm-remarkutil count %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=COUNTBITSTREAM
1116

1217
; YAMLPARSER: error: document root is not of mapping type.
1318

0 commit comments

Comments
 (0)