Skip to content

Commit 795b17d

Browse files
kparzyszkparzysz
authored
[Frontend][OpenMP] Implement directive name parser (#146776)
Implement a state machine that consumes tokens (words delimited by white space), and returns the corresponding directive id, or fails if the tokens did not form a valid name.
1 parent 1aa3969 commit 795b17d

File tree

5 files changed

+333
-0
lines changed

5 files changed

+333
-0
lines changed

‎llvm/include/llvm/Frontend/OpenMP/DirectiveNameParser.h

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
//===- DirectiveNameParser.h ------------------------------------- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_FRONTEND_OPENMP_DIRECTIVENAMEPARSER_H
10+
#define LLVM_FRONTEND_OPENMP_DIRECTIVENAMEPARSER_H
11+
12+
#include "llvm/ADT/SmallVector.h"
13+
#include "llvm/ADT/StringMap.h"
14+
#include "llvm/ADT/StringRef.h"
15+
#include "llvm/Frontend/OpenMP/OMP.h"
16+
17+
#include <memory>
18+
19+
namespace llvm::omp {
20+
/// Parser class for OpenMP directive names. It only recognizes names listed
21+
/// in OMP.td, in particular it does not recognize Fortran's end-directives
22+
/// if they are not explicitly listed in OMP.td.
23+
///
24+
/// The class itself may be a singleton, once it's constructed it never
25+
/// changes.
26+
///
27+
/// Usage:
28+
/// {
29+
/// DirectiveNameParser Parser; // Could be static const.
30+
///
31+
/// DirectiveNameParser::State *S = Parser.initial();
32+
/// for (StringRef Token : Tokens)
33+
/// S = Parser.consume(S, Token); // Passing nullptr is ok.
34+
///
35+
/// if (S == nullptr) {
36+
/// // Error: ended up in a state from which there is no possible path
37+
/// // to a successful parse.
38+
/// } else if (S->Value == OMPD_unknown) {
39+
/// // Parsed a sequence of tokens that are not a complete name, but
40+
/// // parsing more tokens could lead to a successful parse.
41+
/// } else {
42+
/// // Success.
43+
/// ParsedId = S->Value;
44+
/// }
45+
/// }
46+
struct DirectiveNameParser {
47+
DirectiveNameParser(SourceLanguage L = SourceLanguage::C);
48+
49+
struct State {
50+
Directive Value = Directive::OMPD_unknown;
51+
52+
private:
53+
using TransitionMapTy = StringMap<State>;
54+
std::unique_ptr<TransitionMapTy> Transition;
55+
56+
State *next(StringRef Tok);
57+
const State *next(StringRef Tok) const;
58+
bool isValid() const {
59+
return Value != Directive::OMPD_unknown || !Transition->empty();
60+
}
61+
friend struct DirectiveNameParser;
62+
};
63+
64+
const State *initial() const { return &InitialState; }
65+
const State *consume(const State *Current, StringRef Tok) const;
66+
67+
static SmallVector<StringRef> tokenize(StringRef N);
68+
69+
private:
70+
void insertName(StringRef Name, Directive D);
71+
State *insertTransition(State *From, StringRef Tok);
72+
73+
State InitialState;
74+
};
75+
} // namespace llvm::omp
76+
77+
#endif // LLVM_FRONTEND_OPENMP_DIRECTIVENAMEPARSER_H

‎llvm/lib/Frontend/OpenMP/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMFrontendOpenMP
22
OMP.cpp
33
OMPContext.cpp
44
OMPIRBuilder.cpp
5+
DirectiveNameParser.cpp
56

67
ADDITIONAL_HEADER_DIRS
78
${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend

‎llvm/lib/Frontend/OpenMP/DirectiveNameParser.cpp

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
//===- DirectiveNameParser.cpp --------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "llvm/Frontend/OpenMP/DirectiveNameParser.h"
10+
#include "llvm/ADT/Sequence.h"
11+
#include "llvm/ADT/StringExtras.h"
12+
#include "llvm/ADT/StringRef.h"
13+
#include "llvm/Frontend/OpenMP/OMP.h"
14+
15+
#include <cassert>
16+
#include <memory>
17+
18+
namespace llvm::omp {
19+
DirectiveNameParser::DirectiveNameParser(SourceLanguage L) {
20+
// Take every directive, get its name in every version, break the name up
21+
// into whitespace-separated tokens, and insert each token.
22+
for (size_t I : llvm::seq<size_t>(Directive_enumSize)) {
23+
auto D = static_cast<Directive>(I);
24+
if (D == Directive::OMPD_unknown || !(getDirectiveLanguages(D) & L))
25+
continue;
26+
for (unsigned Ver : getOpenMPVersions())
27+
insertName(getOpenMPDirectiveName(D, Ver), D);
28+
}
29+
}
30+
31+
const DirectiveNameParser::State *
32+
DirectiveNameParser::consume(const State *Current, StringRef Tok) const {
33+
if (!Current)
34+
return Current;
35+
assert(Current->isValid() && "Invalid input state");
36+
if (const State *Next = Current->next(Tok))
37+
return Next->isValid() ? Next : nullptr;
38+
return nullptr;
39+
}
40+
41+
SmallVector<StringRef> DirectiveNameParser::tokenize(StringRef Str) {
42+
SmallVector<StringRef> Tokens;
43+
SplitString(Str, Tokens);
44+
return Tokens;
45+
}
46+
47+
void DirectiveNameParser::insertName(StringRef Name, Directive D) {
48+
State *Where = &InitialState;
49+
50+
for (StringRef Tok : tokenize(Name))
51+
Where = insertTransition(Where, Tok);
52+
53+
Where->Value = D;
54+
}
55+
56+
DirectiveNameParser::State *
57+
DirectiveNameParser::insertTransition(State *From, StringRef Tok) {
58+
assert(From && "Expecting state");
59+
if (!From->Transition)
60+
From->Transition = std::make_unique<State::TransitionMapTy>();
61+
if (State *Next = From->next(Tok))
62+
return Next;
63+
64+
auto [Where, DidIt] = From->Transition->try_emplace(Tok, State());
65+
assert(DidIt && "Map insertion failed");
66+
return &Where->second;
67+
}
68+
69+
const DirectiveNameParser::State *
70+
DirectiveNameParser::State::next(StringRef Tok) const {
71+
if (!Transition)
72+
return nullptr;
73+
auto F = Transition->find(Tok);
74+
return F != Transition->end() ? &F->second : nullptr;
75+
}
76+
77+
DirectiveNameParser::State *DirectiveNameParser::State::next(StringRef Tok) {
78+
if (!Transition)
79+
return nullptr;
80+
auto F = Transition->find(Tok);
81+
return F != Transition->end() ? &F->second : nullptr;
82+
}
83+
} // namespace llvm::omp

‎llvm/unittests/Frontend/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ add_llvm_unittest(LLVMFrontendTests
2020
OpenMPCompositionTest.cpp
2121
OpenMPDecompositionTest.cpp
2222
OpenMPDirectiveNameTest.cpp
23+
OpenMPDirectiveNameParserTest.cpp
2324

2425
DEPENDS
2526
acc_gen

‎llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
//===- llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp ----------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "llvm/ADT/Sequence.h"
10+
#include "llvm/ADT/SmallVector.h"
11+
#include "llvm/ADT/StringRef.h"
12+
#include "llvm/Frontend/OpenMP/DirectiveNameParser.h"
13+
#include "llvm/Frontend/OpenMP/OMP.h"
14+
#include "gtest/gtest.h"
15+
16+
#include <cctype>
17+
#include <sstream>
18+
#include <string>
19+
#include <tuple>
20+
#include <vector>
21+
22+
using namespace llvm;
23+
24+
static const omp::DirectiveNameParser &getParser() {
25+
static omp::DirectiveNameParser Parser(omp::SourceLanguage::C |
26+
omp::SourceLanguage::Fortran);
27+
return Parser;
28+
}
29+
30+
static std::vector<std::string> tokenize(StringRef S) {
31+
std::vector<std::string> Tokens;
32+
33+
using TokenIterator = std::istream_iterator<std::string>;
34+
std::string Copy = S.str();
35+
std::istringstream Stream(Copy);
36+
37+
for (auto I = TokenIterator(Stream), E = TokenIterator(); I != E; ++I)
38+
Tokens.push_back(*I);
39+
return Tokens;
40+
}
41+
42+
static std::string &prepareParamName(std::string &Name) {
43+
for (size_t I = 0, E = Name.size(); I != E; ++I) {
44+
// The parameter name must only have alphanumeric characters.
45+
if (!isalnum(Name[I]))
46+
Name[I] = 'X';
47+
}
48+
return Name;
49+
}
50+
51+
namespace llvm {
52+
template <> struct enum_iteration_traits<omp::Directive> {
53+
static constexpr bool is_iterable = true;
54+
};
55+
} // namespace llvm
56+
57+
// Test tokenizing.
58+
59+
class Tokenize : public testing::TestWithParam<omp::Directive> {};
60+
61+
static bool isEqual(const SmallVector<StringRef> &A,
62+
const std::vector<std::string> &B) {
63+
if (A.size() != B.size())
64+
return false;
65+
66+
for (size_t I = 0, E = A.size(); I != E; ++I) {
67+
if (A[I] != StringRef(B[I]))
68+
return false;
69+
}
70+
return true;
71+
}
72+
73+
TEST_P(Tokenize, T) {
74+
omp::Directive DirId = GetParam();
75+
StringRef Name = omp::getOpenMPDirectiveName(DirId, omp::FallbackVersion);
76+
77+
SmallVector<StringRef> tokens1 = omp::DirectiveNameParser::tokenize(Name);
78+
std::vector<std::string> tokens2 = tokenize(Name);
79+
ASSERT_TRUE(isEqual(tokens1, tokens2));
80+
}
81+
82+
static std::string
83+
getParamName1(const testing::TestParamInfo<Tokenize::ParamType> &Info) {
84+
omp::Directive DirId = Info.param;
85+
std::string Name =
86+
omp::getOpenMPDirectiveName(DirId, omp::FallbackVersion).str();
87+
return prepareParamName(Name);
88+
}
89+
90+
INSTANTIATE_TEST_SUITE_P(
91+
DirectiveNameParserTest, Tokenize,
92+
testing::ValuesIn(
93+
llvm::enum_seq(static_cast<omp::Directive>(0),
94+
static_cast<omp::Directive>(omp::Directive_enumSize))),
95+
getParamName1);
96+
97+
// Test parsing of valid names.
98+
99+
using ValueType = std::tuple<omp::Directive, unsigned>;
100+
101+
class ParseValid : public testing::TestWithParam<ValueType> {};
102+
103+
TEST_P(ParseValid, T) {
104+
auto [DirId, Version] = GetParam();
105+
if (DirId == omp::Directive::OMPD_unknown)
106+
return;
107+
108+
std::string Name = omp::getOpenMPDirectiveName(DirId, Version).str();
109+
110+
// Tokenize and parse
111+
auto &Parser = getParser();
112+
auto *State = Parser.initial();
113+
ASSERT_TRUE(State != nullptr);
114+
115+
std::vector<std::string> Tokens = tokenize(Name);
116+
for (auto &Tok : Tokens) {
117+
State = Parser.consume(State, Tok);
118+
ASSERT_TRUE(State != nullptr);
119+
}
120+
121+
ASSERT_EQ(State->Value, DirId);
122+
}
123+
124+
static std::string
125+
getParamName2(const testing::TestParamInfo<ParseValid::ParamType> &Info) {
126+
auto [DirId, Version] = Info.param;
127+
std::string Name = omp::getOpenMPDirectiveName(DirId, Version).str() + "v" +
128+
std::to_string(Version);
129+
return prepareParamName(Name);
130+
}
131+
132+
INSTANTIATE_TEST_SUITE_P(
133+
DirectiveNameParserTest, ParseValid,
134+
testing::Combine(testing::ValuesIn(llvm::enum_seq(
135+
static_cast<omp::Directive>(0),
136+
static_cast<omp::Directive>(omp::Directive_enumSize))),
137+
testing::ValuesIn(omp::getOpenMPVersions())),
138+
getParamName2);
139+
140+
// Test parsing of invalid names
141+
142+
class ParseInvalid : public testing::TestWithParam<std::string> {};
143+
144+
TEST_P(ParseInvalid, T) {
145+
std::string Name = GetParam();
146+
147+
auto &Parser = getParser();
148+
auto *State = Parser.initial();
149+
ASSERT_TRUE(State != nullptr);
150+
151+
std::vector<std::string> Tokens = tokenize(Name);
152+
for (auto &Tok : Tokens)
153+
State = Parser.consume(State, Tok);
154+
155+
ASSERT_TRUE(State == nullptr || State->Value == omp::Directive::OMPD_unknown);
156+
}
157+
158+
namespace {
159+
using namespace std;
160+
161+
INSTANTIATE_TEST_SUITE_P(DirectiveNameParserTest, ParseInvalid,
162+
testing::Values(
163+
// Names that contain invalid tokens
164+
"bad"s, "target teams invalid"s,
165+
"target sections parallel"s,
166+
"target teams distribute parallel for wrong"s,
167+
// Valid beginning, but not a complete name
168+
"begin declare"s,
169+
// Complete name with extra tokens
170+
"distribute simd target"s));
171+
} // namespace

0 commit comments

Comments
 (0)