Skip to content

Commit 4e97064

Browse files
authored
Merge pull request #9112 from AlexDenisov/alexdenisov/introduce-dispatcher
Swift: introduce dispatcher
2 parents d531631 + 43199fa commit 4e97064

File tree

19 files changed

+375
-40
lines changed

19 files changed

+375
-40
lines changed

swift/codegen/cppgen.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ def generate(opts, renderer):
6666
processor = Processor({cls.name: cls for cls in schema.load(opts.schema).classes}, opts.trap_affix)
6767
out = opts.cpp_output
6868
renderer.render(cpp.ClassList(processor.get_classes(), opts.cpp_namespace, opts.trap_affix,
69-
opts.cpp_include_dir), out / f"{opts.trap_affix}Classes.h")
69+
opts.cpp_include_dir, opts.schema), out / f"{opts.trap_affix}Classes.h")
7070

7171

7272
tags = ("cpp", "schema")

swift/codegen/lib/cpp.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ class TrapList:
104104
namespace: str
105105
trap_affix: str
106106
include_dir: str
107+
source: str
107108

108109

109110
@dataclass
@@ -112,6 +113,7 @@ class TagList:
112113

113114
tags: List[Tag]
114115
namespace: str
116+
source: str
115117

116118

117119
@dataclass
@@ -150,3 +152,4 @@ class ClassList:
150152
namespace: str
151153
trap_affix: str
152154
include_dir: str
155+
source: str

swift/codegen/templates/cpp_classes.mustache

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// generated by {{generator}}
1+
// generated by {{generator}} from {{source}}
22
// clang-format off
33
#pragma once
44

swift/codegen/templates/trap_tags.mustache

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// generated by {{generator}}
1+
// generated by {{generator}} from {{source}}
22
// clang-format off
33
#pragma once
44

swift/codegen/templates/trap_traps.mustache

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// generated by {{generator}}
1+
// generated by {{generator}} from {{source}}
22
// clang-format off
33
#pragma once
44

@@ -29,12 +29,5 @@ inline std::ostream &operator<<(std::ostream &out, const {{name}}{{trap_affix}}
2929
<< {{#get_streamer}}e.{{field_name}}{{/get_streamer}}{{/fields}} << ")";
3030
return out;
3131
}
32-
{{#id}}
33-
34-
template <>
35-
struct TagToBindingTrapFunctor<typename {{type}}::Tag> {
36-
using type = {{name}}{{trap_affix}};
37-
};
38-
{{/id}}
3932
{{/traps}}
4033
}

swift/codegen/trapgen.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ def generate(opts, renderer):
6868
for d in e.rhs:
6969
tag_graph.setdefault(d.type, set()).add(e.lhs)
7070

71-
renderer.render(cpp.TrapList(traps, opts.cpp_namespace, opts.trap_affix, opts.cpp_include_dir),
71+
renderer.render(cpp.TrapList(traps, opts.cpp_namespace, opts.trap_affix, opts.cpp_include_dir, opts.dbscheme),
7272
out / f"{opts.trap_affix}Entries.h")
7373

7474
tags = []
@@ -79,7 +79,7 @@ def generate(opts, renderer):
7979
index=index,
8080
id=tag,
8181
))
82-
renderer.render(cpp.TagList(tags, opts.cpp_namespace), out / f"{opts.trap_affix}Tags.h")
82+
renderer.render(cpp.TagList(tags, opts.cpp_namespace, opts.dbscheme), out / f"{opts.trap_affix}Tags.h")
8383

8484

8585
tags = ("cpp", "dbscheme")

swift/extractor/BUILD.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ swift_cc_binary(
66
"SwiftExtractor.cpp",
77
"SwiftExtractor.h",
88
"SwiftExtractorConfiguration.h",
9+
"SwiftDispatcher.h",
10+
"SwiftTagTraits.h",
911
"main.cpp",
1012
],
1113
visibility = ["//swift:__pkg__"],

swift/extractor/SwiftDispatcher.h

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
#pragma once
2+
3+
#include "swift/extractor/trap/TrapArena.h"
4+
#include "swift/extractor/trap/TrapLabelStore.h"
5+
#include "swift/extractor/trap/generated/TrapClasses.h"
6+
#include "swift/extractor/SwiftTagTraits.h"
7+
#include <swift/AST/SourceFile.h>
8+
#include <swift/Basic/SourceManager.h>
9+
#include <llvm/Support/FileSystem.h>
10+
11+
namespace codeql {
12+
13+
namespace detail {
14+
15+
// The following `getKindName`s are used within "TBD" TRAP entries to visually mark an AST node as
16+
// not properly emitted yet.
17+
// TODO: To be replaced with QL counterpart
18+
template <typename Parent, typename Kind>
19+
inline std::string getKindName(Kind kind) {
20+
return Parent::getKindName(kind).str();
21+
}
22+
23+
template <>
24+
inline std::string getKindName<swift::TypeBase, swift::TypeKind>(swift::TypeKind kind) {
25+
switch (kind) {
26+
#define TYPE(CLASS, PARENT) \
27+
case swift::TypeKind::CLASS: \
28+
return #CLASS;
29+
#include "swift/AST/TypeNodes.def"
30+
default:
31+
return "Unknown";
32+
}
33+
}
34+
35+
template <>
36+
std::string inline getKindName<swift::TypeRepr, swift::TypeReprKind>(swift::TypeReprKind kind) {
37+
switch (kind) {
38+
#define TYPEREPR(CLASS, PARENT) \
39+
case swift::TypeReprKind::CLASS: \
40+
return #CLASS;
41+
#include "swift/AST/TypeReprNodes.def"
42+
default:
43+
return "Unknown";
44+
}
45+
}
46+
47+
} // namespace detail
48+
49+
// The main reponsibilities of the SwiftDispatcher are as follows:
50+
// * redirect specific AST node emission to a corresponding visitor (statements, expressions, etc.)
51+
// * storing TRAP labels for emitted AST nodes (in the TrapLabelStore) to avoid re-emission
52+
// Since SwiftDispatcher sees all the AST nodes, it also attaches a location to every 'locatable'
53+
// node (AST nodes that are not types: declarations, statements, expressions, etc.).
54+
class SwiftDispatcher {
55+
public:
56+
// sourceManager, arena, and trap are supposed to outlive the SwiftDispatcher
57+
SwiftDispatcher(const swift::SourceManager& sourceManager, TrapArena& arena, TrapOutput& trap)
58+
: sourceManager{sourceManager}, arena{arena}, trap{trap} {}
59+
60+
template <typename T>
61+
void extract(T* entity) {
62+
fetchLabel(entity);
63+
}
64+
65+
private:
66+
// This method gives a TRAP label for already emitted AST node.
67+
// If the AST node was not emitted yet, then the emission is dispatched to a corresponding
68+
// visitor (see `visit(T *)` methods below).
69+
template <typename E>
70+
TrapLabel<ToTag<E>> fetchLabel(E* e) {
71+
// this is required so we avoid any recursive loop: a `fetchLabel` during the visit of `e` might
72+
// end up calling `fetchLabel` on `e` itself, so we want the visit of `e` to call `fetchLabel`
73+
// only after having called `assignNewLabel` on `e`
74+
assert(!waitingForNewLabel && "fetchLabel called before assignNewLabel");
75+
if (auto l = store.get(e)) {
76+
return *l;
77+
}
78+
waitingForNewLabel = getCanonicalPointer(e);
79+
visit(e);
80+
if (auto l = store.get(e)) {
81+
if constexpr (!std::is_base_of_v<swift::TypeBase, E>) {
82+
attachLocation(e, *l);
83+
}
84+
return *l;
85+
}
86+
assert(!"assignNewLabel not called during visit");
87+
return {};
88+
}
89+
90+
// Due to the lazy emission approach, we must assign a label to a corresponding AST node before
91+
// it actually gets emitted to handle recursive cases such as recursive calls, or recursive type
92+
// declarations
93+
template <typename E>
94+
TrapLabel<ToTag<E>> assignNewLabel(E* e) {
95+
assert(waitingForNewLabel == getCanonicalPointer(e) && "assignNewLabel called on wrong entity");
96+
auto label = getLabel<ToTag<E>>();
97+
trap.assignStar(label);
98+
store.insert(e, label);
99+
waitingForNewLabel = nullptr;
100+
return label;
101+
}
102+
103+
template <typename Tag>
104+
TrapLabel<Tag> getLabel() {
105+
return arena.allocateLabel<Tag>();
106+
}
107+
108+
// This is a helper method to emit TRAP entries for AST nodes that we don't fully support yet.
109+
template <typename Parent, typename Child>
110+
void TBD(Child* entity, const std::string& suffix) {
111+
using namespace std::string_literals;
112+
auto label = assignNewLabel(entity);
113+
auto kind = detail::getKindName<Parent>(static_cast<const Parent*>(entity)->getKind());
114+
auto name = "TBD ("s + kind + suffix + ")";
115+
if constexpr (std::is_same_v<Parent, swift::TypeBase>) {
116+
trap.emit(UnknownTypesTrap{label, name});
117+
} else {
118+
trap.emit(UnknownAstNodesTrap{label, name});
119+
}
120+
}
121+
122+
template <typename Locatable>
123+
void attachLocation(Locatable locatable, TrapLabel<LocatableTag> locatableLabel) {
124+
attachLocation(&locatable, locatableLabel);
125+
}
126+
127+
// Emits a Location TRAP entry and attaches it to an AST node
128+
template <typename Locatable>
129+
void attachLocation(Locatable* locatable, TrapLabel<LocatableTag> locatableLabel) {
130+
auto start = locatable->getStartLoc();
131+
auto end = locatable->getEndLoc();
132+
if (!start.isValid() || !end.isValid()) {
133+
// invalid locations seem to come from entities synthesized by the compiler
134+
return;
135+
}
136+
std::string filepath = getFilepath(start);
137+
auto fileLabel = arena.allocateLabel<FileTag>();
138+
trap.assignKey(fileLabel, filepath);
139+
// TODO: do not emit duplicate trap entries for Files
140+
trap.emit(FilesTrap{fileLabel, filepath});
141+
auto [startLine, startColumn] = sourceManager.getLineAndColumnInBuffer(start);
142+
auto [endLine, endColumn] = sourceManager.getLineAndColumnInBuffer(end);
143+
auto locLabel = arena.allocateLabel<LocationTag>();
144+
trap.assignKey(locLabel, '{', fileLabel, "}:", startLine, ':', startColumn, ':', endLine, ':',
145+
endColumn);
146+
trap.emit(LocationsTrap{locLabel, fileLabel, startLine, startColumn, endLine, endColumn});
147+
trap.emit(LocatablesTrap{locatableLabel, locLabel});
148+
}
149+
150+
std::string getFilepath(swift::SourceLoc loc) {
151+
// TODO: this needs more testing
152+
// TODO: check canonicaliztion of names on a case insensitive filesystems
153+
// TODO: make symlink resolution conditional on CODEQL_PRESERVE_SYMLINKS=true
154+
auto displayName = sourceManager.getDisplayNameForLoc(loc);
155+
llvm::SmallString<PATH_MAX> realPath;
156+
if (std::error_code ec = llvm::sys::fs::real_path(displayName, realPath)) {
157+
std::cerr << "Cannot get real path: '" << displayName.str() << "': " << ec.message() << "\n";
158+
return {};
159+
}
160+
return realPath.str().str();
161+
}
162+
163+
// TODO: The following methods are supposed to redirect TRAP emission to correpsonding visitors,
164+
// which are to be introduced in follow-up PRs
165+
void visit(swift::Decl* decl) { TBD<swift::Decl>(decl, "Decl"); }
166+
void visit(swift::Stmt* stmt) { TBD<swift::Stmt>(stmt, "Stmt"); }
167+
void visit(swift::Expr* expr) { TBD<swift::Expr>(expr, "Expr"); }
168+
void visit(swift::Pattern* pattern) { TBD<swift::Pattern>(pattern, "Pattern"); }
169+
void visit(swift::TypeRepr* type) { TBD<swift::TypeRepr>(type, "TypeRepr"); }
170+
void visit(swift::TypeBase* type) { TBD<swift::TypeBase>(type, "Type"); }
171+
172+
const swift::SourceManager& sourceManager;
173+
TrapArena& arena;
174+
TrapOutput& trap;
175+
TrapLabelStore store;
176+
const void* waitingForNewLabel{nullptr};
177+
};
178+
179+
} // namespace codeql

swift/extractor/SwiftExtractor.cpp

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,15 @@
1212
#include <llvm/Support/FileSystem.h>
1313
#include <llvm/Support/Path.h>
1414

15-
#include "swift/extractor/trap/TrapClasses.h"
16-
#include "swift/extractor/trap/TrapArena.h"
15+
#include "swift/extractor/trap/generated/TrapClasses.h"
1716
#include "swift/extractor/trap/TrapOutput.h"
17+
#include "swift/extractor/SwiftDispatcher.h"
1818

1919
using namespace codeql;
2020

21-
static void extractFile(const SwiftExtractorConfiguration& config, swift::SourceFile& file) {
21+
static void extractFile(const SwiftExtractorConfiguration& config,
22+
swift::CompilerInstance& compiler,
23+
swift::SourceFile& file) {
2224
if (std::error_code ec = llvm::sys::fs::create_directories(config.trapDir)) {
2325
std::cerr << "Cannot create TRAP directory: " << ec.message() << "\n";
2426
return;
@@ -79,12 +81,17 @@ static void extractFile(const SwiftExtractorConfiguration& config, swift::Source
7981

8082
TrapOutput trap{trapStream};
8183
TrapArena arena{};
82-
auto label = arena.allocateLabel<FileTag>();
83-
trap.assignStar(label);
84-
File f{};
85-
f.id = label;
86-
f.name = srcFilePath.str().str();
87-
trap.emit(f);
84+
85+
// In the case of emtpy files, the dispatcher is not called, but we still want to 'record' the
86+
// fact that the file was extracted
87+
auto fileLabel = arena.allocateLabel<FileTag>();
88+
trap.assignKey(fileLabel, srcFilePath.str().str());
89+
trap.emit(FilesTrap{fileLabel, srcFilePath.str().str()});
90+
91+
SwiftDispatcher dispatcher(compiler.getSourceMgr(), arena, trap);
92+
for (swift::Decl* decl : file.getTopLevelDecls()) {
93+
dispatcher.extract(decl);
94+
}
8895

8996
// TODO: Pick a better name to avoid collisions
9097
std::string trapName = file.getFilename().str() + ".trap";
@@ -108,11 +115,11 @@ void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
108115
module->getFiles().front()->getKind() == swift::FileUnitKind::Source) {
109116
// We can only call getMainSourceFile if the first file is of a Source kind
110117
swift::SourceFile& file = module->getMainSourceFile();
111-
extractFile(config, file);
118+
extractFile(config, compiler, file);
112119
}
113120
} else {
114121
for (auto s : compiler.getPrimarySourceFiles()) {
115-
extractFile(config, *s);
122+
extractFile(config, compiler, *s);
116123
}
117124
}
118125
}

swift/extractor/SwiftTagTraits.h

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#pragma once
2+
3+
// This file implements the mapping needed by the API defined in the TrapTagTraits.h
4+
#include <swift/AST/ASTVisitor.h>
5+
#include "swift/extractor/trap/TrapTagTraits.h"
6+
#include "swift/extractor/trap/generated/TrapTags.h"
7+
8+
namespace codeql {
9+
10+
// codegen goes with QL acronym convention (Sil instead of SIL), we need to remap it to Swift's
11+
// convention
12+
using SILBlockStorageTypeTag = SilBlockStorageTypeTag;
13+
using SILBoxTypeTag = SilBoxTypeTag;
14+
using SILFunctionTypeTag = SilFunctionTypeTag;
15+
using SILTokenTypeTag = SilTokenTypeTag;
16+
17+
#define MAP_TYPE_TO_TAG(TYPE, TAG) \
18+
template <> \
19+
struct detail::ToTagFunctor<swift::TYPE> { \
20+
using type = TAG; \
21+
}
22+
#define MAP_TAG(TYPE) MAP_TYPE_TO_TAG(TYPE, TYPE##Tag)
23+
#define MAP_SUBTAG(TYPE, PARENT) \
24+
MAP_TAG(TYPE); \
25+
static_assert(std::is_base_of_v<PARENT##Tag, TYPE##Tag>, \
26+
#PARENT "Tag must be a base of " #TYPE "Tag");
27+
28+
#define OVERRIDE_TAG(TYPE, TAG) \
29+
template <> \
30+
struct detail::ToTagOverride<swift::TYPE> { \
31+
using type = TAG; \
32+
}; \
33+
static_assert(std::is_base_of_v<TYPE##Tag, TAG>, "override is not a subtag");
34+
35+
MAP_TAG(Stmt);
36+
#define ABSTRACT_STMT(CLASS, PARENT) MAP_SUBTAG(CLASS##Stmt, PARENT)
37+
#define STMT(CLASS, PARENT) ABSTRACT_STMT(CLASS, PARENT)
38+
#include "swift/AST/StmtNodes.def"
39+
40+
MAP_TAG(Expr);
41+
#define ABSTRACT_EXPR(CLASS, PARENT) MAP_SUBTAG(CLASS##Expr, PARENT)
42+
#define EXPR(CLASS, PARENT) ABSTRACT_EXPR(CLASS, PARENT)
43+
#include "swift/AST/ExprNodes.def"
44+
45+
MAP_TAG(Decl);
46+
#define ABSTRACT_DECL(CLASS, PARENT) MAP_SUBTAG(CLASS##Decl, PARENT)
47+
#define DECL(CLASS, PARENT) ABSTRACT_DECL(CLASS, PARENT)
48+
#include "swift/AST/DeclNodes.def"
49+
50+
MAP_TAG(Pattern);
51+
#define ABSTRACT_PATTERN(CLASS, PARENT) MAP_SUBTAG(CLASS##Pattern, PARENT)
52+
#define PATTERN(CLASS, PARENT) ABSTRACT_PATTERN(CLASS, PARENT)
53+
#include "swift/AST/PatternNodes.def"
54+
55+
MAP_TAG(TypeRepr);
56+
MAP_TYPE_TO_TAG(TypeBase, TypeTag);
57+
#define ABSTRACT_TYPE(CLASS, PARENT) MAP_SUBTAG(CLASS##Type, PARENT)
58+
#define TYPE(CLASS, PARENT) ABSTRACT_TYPE(CLASS, PARENT)
59+
#include "swift/AST/TypeNodes.def"
60+
61+
OVERRIDE_TAG(FuncDecl, ConcreteFuncDeclTag);
62+
OVERRIDE_TAG(VarDecl, ConcreteVarDeclTag);
63+
64+
#undef MAP_TAG
65+
#undef MAP_SUBTAG
66+
#undef MAP_TYPE_TO_TAG
67+
#undef OVERRIDE_TAG
68+
69+
// All the other macros defined here are undefined by the .def files
70+
71+
} // namespace codeql

0 commit comments

Comments
 (0)