Skip to content

Commit 15c3793

Browse files
cyndyishidajansvoboda11
cyndyishida
and
jansvoboda11
authored
[clang][scan-deps] Report a scanned TU's visible modules (#147969)
Clients of the dependency scanning service may need to add dependencies based on the visibility of importing modules, for example, when determining whether a Swift overlay dependency should be brought in based on whether there's a corresponding **visible** clang module for it. This patch introduces a new field `VisibleModules` that contains all the visible top-level modules in a given TU. Because visibility is determined by which headers or (sub)modules were imported, and not top-level module dependencies, the scanner now performs a separate DFS starting from what was directly imported for this computation. In my local performance testing, there was no observable performance impact. resolves: rdar://151416358 --------- Co-authored-by: Jan Svoboda <jan@svoboda.ai>
1 parent 6882a30 commit 15c3793

File tree

8 files changed

+201
-26
lines changed

8 files changed

+201
-26
lines changed

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ struct TranslationUnitDeps {
5757
/// determined that the differences are benign for this compilation.
5858
std::vector<ModuleID> ClangModuleDeps;
5959

60+
/// A list of module names that are visible to this translation unit. This
61+
/// includes both direct and transitive module dependencies.
62+
std::vector<std::string> VisibleModules;
63+
6064
/// A list of the C++20 named modules this translation unit depends on.
6165
std::vector<std::string> NamedModuleDeps;
6266

@@ -150,7 +154,7 @@ class DependencyScanningTool {
150154
/// Given a compilation context specified via the Clang driver command-line,
151155
/// gather modular dependencies of module with the given name, and return the
152156
/// information needed for explicit build.
153-
llvm::Expected<ModuleDepsGraph> getModuleDependencies(
157+
llvm::Expected<TranslationUnitDeps> getModuleDependencies(
154158
StringRef ModuleName, const std::vector<std::string> &CommandLine,
155159
StringRef CWD, const llvm::DenseSet<ModuleID> &AlreadySeen,
156160
LookupModuleOutputCallback LookupModuleOutput);
@@ -188,6 +192,10 @@ class FullDependencyConsumer : public DependencyConsumer {
188192
DirectModuleDeps.push_back(ID);
189193
}
190194

195+
void handleVisibleModule(std::string ModuleName) override {
196+
VisibleModules.push_back(ModuleName);
197+
}
198+
191199
void handleContextHash(std::string Hash) override {
192200
ContextHash = std::move(Hash);
193201
}
@@ -201,7 +209,6 @@ class FullDependencyConsumer : public DependencyConsumer {
201209
}
202210

203211
TranslationUnitDeps takeTranslationUnitDeps();
204-
ModuleDepsGraph takeModuleGraphDeps();
205212

206213
private:
207214
std::vector<std::string> Dependencies;
@@ -210,6 +217,7 @@ class FullDependencyConsumer : public DependencyConsumer {
210217
std::string ModuleName;
211218
std::vector<std::string> NamedModuleDeps;
212219
std::vector<ModuleID> DirectModuleDeps;
220+
std::vector<std::string> VisibleModules;
213221
std::vector<Command> Commands;
214222
std::string ContextHash;
215223
std::vector<std::string> OutputPaths;

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ class DependencyConsumer {
5959

6060
virtual void handleDirectModuleDependency(ModuleID MD) = 0;
6161

62+
virtual void handleVisibleModule(std::string ModuleName) = 0;
63+
6264
virtual void handleContextHash(std::string Hash) = 0;
6365
};
6466

‎clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,11 @@ class ModuleDepCollector final : public DependencyCollector {
323323
llvm::MapVector<const Module *, PrebuiltModuleDep> DirectPrebuiltModularDeps;
324324
/// Working set of direct modular dependencies.
325325
llvm::SetVector<const Module *> DirectModularDeps;
326+
/// Working set of direct modular dependencies, as they were imported.
327+
llvm::SmallPtrSet<const Module *, 32> DirectImports;
328+
/// All direct and transitive visible modules.
329+
llvm::StringSet<> VisibleModules;
330+
326331
/// Options that control the dependency output generation.
327332
std::unique_ptr<DependencyOutputOptions> Opts;
328333
/// A Clang invocation that's based on the original TU invocation and that has
@@ -337,6 +342,9 @@ class ModuleDepCollector final : public DependencyCollector {
337342
/// Checks whether the module is known as being prebuilt.
338343
bool isPrebuiltModule(const Module *M);
339344

345+
/// Computes all visible modules resolved from direct imports.
346+
void addVisibleModules();
347+
340348
/// Adds \p Path to \c FileDeps, making it absolute if necessary.
341349
void addFileDep(StringRef Path);
342350
/// Adds \p Path to \c MD.FileDeps, making it absolute if necessary.

‎clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ class MakeDependencyPrinterConsumer : public DependencyConsumer {
4040
void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {}
4141
void handleModuleDependency(ModuleDeps MD) override {}
4242
void handleDirectModuleDependency(ModuleID ID) override {}
43+
void handleVisibleModule(std::string ModuleName) override {}
4344
void handleContextHash(std::string Hash) override {}
4445

4546
void printDependencies(std::string &S) {
@@ -154,7 +155,8 @@ DependencyScanningTool::getTranslationUnitDependencies(
154155
return Consumer.takeTranslationUnitDeps();
155156
}
156157

157-
llvm::Expected<ModuleDepsGraph> DependencyScanningTool::getModuleDependencies(
158+
llvm::Expected<TranslationUnitDeps>
159+
DependencyScanningTool::getModuleDependencies(
158160
StringRef ModuleName, const std::vector<std::string> &CommandLine,
159161
StringRef CWD, const llvm::DenseSet<ModuleID> &AlreadySeen,
160162
LookupModuleOutputCallback LookupModuleOutput) {
@@ -164,7 +166,7 @@ llvm::Expected<ModuleDepsGraph> DependencyScanningTool::getModuleDependencies(
164166
Controller, ModuleName);
165167
if (Result)
166168
return std::move(Result);
167-
return Consumer.takeModuleGraphDeps();
169+
return Consumer.takeTranslationUnitDeps();
168170
}
169171

170172
TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
@@ -175,6 +177,7 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
175177
TU.NamedModuleDeps = std::move(NamedModuleDeps);
176178
TU.FileDeps = std::move(Dependencies);
177179
TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
180+
TU.VisibleModules = std::move(VisibleModules);
178181
TU.Commands = std::move(Commands);
179182

180183
for (auto &&M : ClangModuleDeps) {
@@ -190,19 +193,4 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
190193
return TU;
191194
}
192195

193-
ModuleDepsGraph FullDependencyConsumer::takeModuleGraphDeps() {
194-
ModuleDepsGraph ModuleGraph;
195-
196-
for (auto &&M : ClangModuleDeps) {
197-
auto &MD = M.second;
198-
// TODO: Avoid handleModuleDependency even being called for modules
199-
// we've already seen.
200-
if (AlreadySeen.count(M.first))
201-
continue;
202-
ModuleGraph.push_back(std::move(MD));
203-
}
204-
205-
return ModuleGraph;
206-
}
207-
208196
CallbackActionController::~CallbackActionController() {}

‎clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -673,8 +673,10 @@ void ModuleDepCollectorPP::handleImport(const Module *Imported) {
673673
if (MDC.isPrebuiltModule(TopLevelModule))
674674
MDC.DirectPrebuiltModularDeps.insert(
675675
{TopLevelModule, PrebuiltModuleDep{TopLevelModule}});
676-
else
676+
else {
677677
MDC.DirectModularDeps.insert(TopLevelModule);
678+
MDC.DirectImports.insert(Imported);
679+
}
678680
}
679681

680682
void ModuleDepCollectorPP::EndOfMainFile() {
@@ -706,6 +708,8 @@ void ModuleDepCollectorPP::EndOfMainFile() {
706708
if (!MDC.isPrebuiltModule(M))
707709
MDC.DirectModularDeps.insert(M);
708710

711+
MDC.addVisibleModules();
712+
709713
for (const Module *M : MDC.DirectModularDeps)
710714
handleTopLevelModule(M);
711715

@@ -727,6 +731,9 @@ void ModuleDepCollectorPP::EndOfMainFile() {
727731
MDC.Consumer.handleDirectModuleDependency(It->second->ID);
728732
}
729733

734+
for (auto &&I : MDC.VisibleModules)
735+
MDC.Consumer.handleVisibleModule(std::string(I.getKey()));
736+
730737
for (auto &&I : MDC.FileDeps)
731738
MDC.Consumer.handleFileDependency(I);
732739

@@ -993,6 +1000,29 @@ bool ModuleDepCollector::isPrebuiltModule(const Module *M) {
9931000
return true;
9941001
}
9951002

1003+
void ModuleDepCollector::addVisibleModules() {
1004+
llvm::DenseSet<const Module *> ImportedModules;
1005+
auto InsertVisibleModules = [&](const Module *M) {
1006+
if (ImportedModules.contains(M))
1007+
return;
1008+
1009+
VisibleModules.insert(M->getTopLevelModuleName());
1010+
SmallVector<Module *> Stack;
1011+
M->getExportedModules(Stack);
1012+
while (!Stack.empty()) {
1013+
const Module *CurrModule = Stack.pop_back_val();
1014+
if (ImportedModules.contains(CurrModule))
1015+
continue;
1016+
ImportedModules.insert(CurrModule);
1017+
VisibleModules.insert(CurrModule->getTopLevelModuleName());
1018+
CurrModule->getExportedModules(Stack);
1019+
}
1020+
};
1021+
1022+
for (const Module *Import : DirectImports)
1023+
InsertVisibleModules(Import);
1024+
}
1025+
9961026
static StringRef makeAbsoluteAndPreferred(CompilerInstance &CI, StringRef Path,
9971027
SmallVectorImpl<char> &Storage) {
9981028
if (llvm::sys::path::is_absolute(Path) &&

‎clang/test/ClangScanDeps/visible-modules.c

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// This test verifies that the modules visible to the translation unit are computed in dependency scanning.
2+
// "client" in the first scan represents the translation unit that imports an explicit submodule,
3+
// that only exports one other module.
4+
// In the second scan, the translation unit that imports an explicit submodule,
5+
// that exports an additional module.
6+
// Thus, the dependencies of the top level module for the submodule always differ from what is visible to the TU.
7+
8+
// RUN: rm -rf %t
9+
// RUN: split-file %s %t
10+
// RUN: sed -e "s|DIR|%/t|g" %t/compile-commands.json.in > %t/compile-commands.json
11+
// RUN: clang-scan-deps -emit-visible-modules -compilation-database %t/compile-commands.json \
12+
// RUN: -j 1 -format experimental-full 2>&1 > %t/result-first-scan.json
13+
// RUN: cat %t/result-first-scan.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix=SINGLE
14+
15+
/// Re-run scan with different module map for direct dependency.
16+
// RUN: mv %t/A_with_visible_export.modulemap %t/Sysroot/usr/include/A/module.modulemap
17+
// RUN: clang-scan-deps -emit-visible-modules -compilation-database %t/compile-commands.json \
18+
// RUN: -j 1 -format experimental-full 2>&1 > %t/result.json
19+
// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix=MULTIPLE
20+
21+
// RUN: %deps-to-rsp %t/result.json --module-name=transitive > %t/transitive.rsp
22+
// RUN: %deps-to-rsp %t/result.json --module-name=visible > %t/visible.rsp
23+
// RUN: %deps-to-rsp %t/result.json --module-name=invisible > %t/invisible.rsp
24+
// RUN: %deps-to-rsp %t/result.json --module-name=A > %t/A.rsp
25+
// RUN: %deps-to-rsp %t/result.json --tu-index=0 > %t/tu.rsp
26+
27+
// RUN: %clang @%t/transitive.rsp
28+
// RUN: %clang @%t/visible.rsp
29+
// RUN: %clang @%t/invisible.rsp
30+
// RUN: %clang @%t/A.rsp
31+
32+
/// Verify compilation & scan agree with each other.
33+
// RUN: not %clang @%t/tu.rsp 2>&1 | FileCheck %s --check-prefix=COMPILE
34+
35+
// SINGLE: "visible-clang-modules": [
36+
// SINGLE-NEXT: "A"
37+
// SINGLE-NEXT: ]
38+
39+
// MULTIPLE: "visible-clang-modules": [
40+
// MULTIPLE-NEXT: "A",
41+
// MULTIPLE-NEXT: "visible"
42+
// MULTIPLE-NEXT: ]
43+
44+
// COMPILE-NOT: 'visible_t' must be declared before it is used
45+
// COMPILE: 'transitive_t' must be declared before it is used
46+
// COMPILE: 'invisible_t' must be declared before it is used
47+
48+
//--- compile-commands.json.in
49+
[
50+
{
51+
"directory": "DIR",
52+
"command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps",
53+
"file": "DIR/client.c"
54+
}
55+
]
56+
57+
//--- Sysroot/usr/include/A/module.modulemap
58+
module A {
59+
explicit module visibleToTU {
60+
header "visibleToTU.h"
61+
}
62+
explicit module invisibleToTU {
63+
header "invisibleToTU.h"
64+
}
65+
}
66+
67+
//--- A_with_visible_export.modulemap
68+
module A {
69+
explicit module visibleToTU {
70+
header "visibleToTU.h"
71+
export visible
72+
}
73+
explicit module invisibleToTU {
74+
header "invisibleToTU.h"
75+
}
76+
}
77+
78+
//--- Sysroot/usr/include/A/visibleToTU.h
79+
#include <visible/visible.h>
80+
typedef int A_visibleToTU;
81+
82+
//--- Sysroot/usr/include/A/invisibleToTU.h
83+
#include <invisible/invisible.h>
84+
typedef int A_invisibleToTU;
85+
86+
//--- Sysroot/usr/include/invisible/module.modulemap
87+
module invisible {
88+
umbrella "."
89+
}
90+
91+
//--- Sysroot/usr/include/invisible/invisible.h
92+
typedef int invisible_t;
93+
94+
//--- Sysroot/usr/include/visible/module.modulemap
95+
module visible {
96+
umbrella "."
97+
}
98+
99+
//--- Sysroot/usr/include/visible/visible.h
100+
#include <transitive/transitive.h>
101+
typedef int visible_t;
102+
103+
//--- Sysroot/usr/include/transitive/module.modulemap
104+
module transitive {
105+
umbrella "."
106+
}
107+
108+
//--- Sysroot/usr/include/transitive/transitive.h
109+
typedef int transitive_t;
110+
111+
//--- client.c
112+
#include <A/visibleToTU.h>
113+
visible_t foo_v(void);
114+
// Both decls are not visible, thus should fail to actually compile.
115+
transitive_t foo_t(void);
116+
invisible_t foo_i(void);

0 commit comments

Comments
 (0)