Skip to content

Commit b75b9f7

Browse files
deprilula28deprilula28
committed
Use TCPP as the pre-processor
1 parent 916bef7 commit b75b9f7

File tree

7 files changed

+174
-89
lines changed

7 files changed

+174
-89
lines changed

‎.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,6 @@
8787
[submodule "3rdparty/dxc/dxc"]
8888
path = 3rdparty/dxc/dxc
8989
url = git@github.com:Devsh-Graphics-Programming/DirectXShaderCompiler.git
90+
[submodule "3rdparty/tcpp"]
91+
path = 3rdparty/tcpp
92+
url = https://github.com/bnoazx005/tcpp

‎3rdparty/tcpp

Submodule tcpp added at b984479

‎include/nbl/asset/utils/CHLSLCompiler.h

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ class NBL_API2 CHLSLCompiler final : public IShaderCompiler
4242
// return "";
4343
//}
4444

45+
std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE stage, const SPreprocessorOptions& preprocessOptions) const override;
46+
4547
protected:
4648

4749
void insertIntoStart(std::string& code, std::ostringstream&& ins) const override;
@@ -63,17 +65,17 @@ class NBL_API2 CHLSLCompiler final : public IShaderCompiler
6365
class DxcCompilationResult
6466
{
6567
public:
66-
std::unique_ptr<IDxcBlobEncoding> errorMessages;
67-
std::unique_ptr<IDxcBlob> objectBlob;
68-
std::unique_ptr<IDxcResult> compileResult;
68+
IDxcBlobEncoding* errorMessages;
69+
IDxcBlob* objectBlob;
70+
IDxcResult* compileResult;
6971

7072
char* GetErrorMessagesString()
7173
{
7274
return reinterpret_cast<char*>(errorMessages->GetBufferPointer());
7375
}
7476
};
7577

76-
DxcCompilationResult dxcCompile(std::string& source, LPCWSTR* args, uint32_t argCount, const SOptions& options) const;
78+
CHLSLCompiler::DxcCompilationResult dxcCompile(std::string& source, LPCWSTR* args, uint32_t argCount, const SOptions& options) const;
7779
};
7880

7981
}

‎include/nbl/asset/utils/IShaderCompiler.h

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
178178
179179
@returns Shader containing logically same High Level code as input but with #include directives resolved.
180180
*/
181-
std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE stage, const SPreprocessorOptions& preprocessOptions) const;
181+
virtual std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE stage, const SPreprocessorOptions& preprocessOptions) const;
182182

183183
std::string preprocessShader(system::IFile* sourcefile, IShader::E_SHADER_STAGE stage, const SPreprocessorOptions& preprocessOptions) const;
184184

@@ -261,6 +261,75 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
261261

262262
const CIncludeFinder* getDefaultIncludeFinder() const { return m_defaultIncludeFinder.get(); }
263263

264+
//string to be replaced with all "#" except those in "#include"
265+
static constexpr const char* PREPROC_DIRECTIVE_DISABLER = "_this_is_a_hash_";
266+
static constexpr const char* PREPROC_DIRECTIVE_ENABLER = PREPROC_DIRECTIVE_DISABLER;
267+
static constexpr const char* PREPROC_GL__DISABLER = "_this_is_a_GL__prefix_";
268+
static constexpr const char* PREPROC_GL__ENABLER = PREPROC_GL__DISABLER;
269+
static constexpr const char* PREPROC_LINE_CONTINUATION_DISABLER = "_this_is_a_line_continuation_\n";
270+
static constexpr const char* PREPROC_LINE_CONTINUATION_ENABLER = "_this_is_a_line_continuation_";
271+
272+
static void disableAllDirectivesExceptIncludes(std::string& _glslCode)
273+
{
274+
// TODO: replace this with a proper-ish proprocessor and includer one day
275+
std::regex directive("#(?!(include|version|pragma shader_stage|line))");//all # not followed by "include" nor "version" nor "pragma shader_stage"
276+
//`#pragma shader_stage(...)` is needed for determining shader stage when `_stage` param of IShaderCompiler functions is set to ESS_UNKNOWN
277+
auto result = std::regex_replace(_glslCode, directive, PREPROC_DIRECTIVE_DISABLER);
278+
std::regex glMacro("[ \t\r\n\v\f]GL_");
279+
result = std::regex_replace(result, glMacro, PREPROC_GL__DISABLER);
280+
std::regex lineContinuation("\\\\[ \t\r\n\v\f]*\n");
281+
_glslCode = std::regex_replace(result, lineContinuation, PREPROC_LINE_CONTINUATION_DISABLER);
282+
}
283+
static void reenableDirectives(std::string& _glslCode)
284+
{
285+
std::regex lineContinuation(PREPROC_LINE_CONTINUATION_ENABLER);
286+
auto result = std::regex_replace(_glslCode, lineContinuation, " \\");
287+
std::regex glMacro(PREPROC_GL__ENABLER);
288+
result = std::regex_replace(result, glMacro, " GL_");
289+
std::regex directive(PREPROC_DIRECTIVE_ENABLER);
290+
_glslCode = std::regex_replace(result, directive, "#");
291+
}
292+
static std::string encloseWithinExtraInclGuards(std::string&& _glslCode, uint32_t _maxInclusions, const char* _identifier)
293+
{
294+
assert(_maxInclusions != 0u);
295+
296+
using namespace std::string_literals;
297+
std::string defBase_ = "_GENERATED_INCLUDE_GUARD_"s + _identifier + "_";
298+
std::replace_if(defBase_.begin(), defBase_.end(), [](char c) ->bool { return !::isalpha(c) && !::isdigit(c); }, '_');
299+
300+
auto genDefs = [&defBase_, _maxInclusions, _identifier] {
301+
auto defBase = [&defBase_](uint32_t n) { return defBase_ + std::to_string(n); };
302+
std::string defs = "#ifndef " + defBase(0) + "\n\t#define " + defBase(0) + "\n";
303+
for (uint32_t i = 1u; i <= _maxInclusions; ++i) {
304+
const std::string defname = defBase(i);
305+
defs += "#elif !defined(" + defname + ")\n\t#define " + defname + "\n";
306+
}
307+
defs += "#endif\n";
308+
return defs;
309+
};
310+
auto genUndefs = [&defBase_, _maxInclusions, _identifier] {
311+
auto defBase = [&defBase_](int32_t n) { return defBase_ + std::to_string(n); };
312+
std::string undefs = "#ifdef " + defBase(_maxInclusions) + "\n\t#undef " + defBase(_maxInclusions) + "\n";
313+
for (int32_t i = _maxInclusions - 1; i >= 0; --i) {
314+
const std::string defname = defBase(i);
315+
undefs += "#elif defined(" + defname + ")\n\t#undef " + defname + "\n";
316+
}
317+
undefs += "#endif\n";
318+
return undefs;
319+
};
320+
321+
return
322+
genDefs() +
323+
"\n"
324+
"#ifndef " + defBase_ + std::to_string(_maxInclusions) +
325+
"\n" +
326+
"#line 1 \"" + _identifier + "\"\n" +
327+
_glslCode +
328+
"\n"
329+
"#endif"
330+
"\n\n" +
331+
genUndefs();
332+
}
264333
protected:
265334

266335
virtual void insertIntoStart(std::string& code, std::ostringstream&& ins) const = 0;
@@ -278,8 +347,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
278347
insertIntoStart(code, std::move(insertion));
279348
}
280349

281-
private:
282350
core::smart_refctd_ptr<system::ISystem> m_system;
351+
private:
283352
core::smart_refctd_ptr<CIncludeFinder> m_defaultIncludeFinder;
284353
};
285354

‎src/nbl/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,8 @@ endif()
470470
## Set up 3rdparty deps
471471
# Parallel Hashmap
472472
target_include_directories(Nabla PUBLIC ${THIRD_PARTY_SOURCE_DIR}/parallel-hashmap/parallel_hashmap)
473+
# TCPP
474+
target_include_directories(Nabla PUBLIC ${THIRD_PARTY_SOURCE_DIR}/tcpp)
473475
# oneDPL for clang C++17 parallelism
474476
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
475477
add_dependencies(Nabla oneDPL)

‎src/nbl/asset/utils/CHLSLCompiler.cpp

Lines changed: 84 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@
1111
#include <dxc/dxcapi.h>
1212
#include <combaseapi.h>
1313

14+
#define TCPP_IMPLEMENTATION
15+
#include <tcpp/source/tcppLibrary.hpp>
16+
#undef TCPP_IMPLEMENTATION
17+
1418
using namespace nbl;
1519
using namespace nbl::asset;
1620

@@ -36,6 +40,49 @@ CHLSLCompiler::~CHLSLCompiler()
3640
m_dxcCompiler->Release();
3741
}
3842

43+
static tcpp::IInputStream* getInputStreamInclude(
44+
const IShaderCompiler::CIncludeFinder* _inclFinder,
45+
const system::ISystem* _fs,
46+
uint32_t _maxInclCnt,
47+
const char* _requesting_source,
48+
bool _type // true for #include "string"; false for #include <string>
49+
)
50+
{
51+
std::string res_str;
52+
53+
std::filesystem::path relDir;
54+
const bool reqFromBuiltin = builtin::hasPathPrefix(_requesting_source);
55+
const bool reqBuiltin = builtin::hasPathPrefix(_requesting_source);
56+
if (!reqFromBuiltin && !reqBuiltin)
57+
{
58+
//While #includ'ing a builtin, one must specify its full path (starting with "nbl/builtin" or "/nbl/builtin").
59+
// This rule applies also while a builtin is #includ`ing another builtin.
60+
//While including a filesystem file it must be either absolute path (or relative to any search dir added to asset::iIncludeHandler; <>-type),
61+
// or path relative to executable's working directory (""-type).
62+
relDir = std::filesystem::path(_requesting_source).parent_path();
63+
}
64+
std::filesystem::path name = _type ? (relDir / _requesting_source) : (_requesting_source);
65+
66+
if (std::filesystem::exists(name) && !reqBuiltin)
67+
name = std::filesystem::absolute(name);
68+
69+
if (_type)
70+
res_str = _inclFinder->getIncludeRelative(relDir, _requesting_source);
71+
else //shaderc_include_type_standard
72+
res_str = _inclFinder->getIncludeStandard(relDir, _requesting_source);
73+
74+
if (!res_str.size()) {
75+
return nullptr;
76+
}
77+
else {
78+
//employ encloseWithinExtraInclGuards() in order to prevent infinite loop of (not necesarilly direct) self-inclusions while other # directives (incl guards among them) are disabled
79+
IShaderCompiler::disableAllDirectivesExceptIncludes(res_str);
80+
res_str = IShaderCompiler::encloseWithinExtraInclGuards(std::move(res_str), _maxInclCnt, name.string().c_str());
81+
}
82+
83+
return new tcpp::StringInputStream(std::move(res_str));
84+
}
85+
3986
CHLSLCompiler::DxcCompilationResult CHLSLCompiler::dxcCompile(std::string& source, LPCWSTR* args, uint32_t argCount, const SOptions& options) const
4087
{
4188
if (options.genDebugInfo)
@@ -77,8 +124,8 @@ CHLSLCompiler::DxcCompilationResult CHLSLCompiler::dxcCompile(std::string& sourc
77124
assert(SUCCEEDED(res));
78125

79126
DxcCompilationResult result;
80-
result.errorMessages = std::unique_ptr<IDxcBlobEncoding>(errorBuffer);
81-
result.compileResult = std::unique_ptr<IDxcResult>(compileResult);
127+
result.errorMessages = errorBuffer;
128+
result.compileResult = compileResult;
82129
result.objectBlob = nullptr;
83130

84131
if (!SUCCEEDED(compilationStatus))
@@ -91,11 +138,42 @@ CHLSLCompiler::DxcCompilationResult CHLSLCompiler::dxcCompile(std::string& sourc
91138
res = compileResult->GetResult(&resultingBlob);
92139
assert(SUCCEEDED(res));
93140

94-
result.objectBlob = std::unique_ptr<IDxcBlob>(resultingBlob);
141+
result.objectBlob = resultingBlob;
95142

96143
return result;
97144
}
98145

146+
std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE stage, const SPreprocessorOptions& preprocessOptions) const
147+
{
148+
if (preprocessOptions.extraDefines.size())
149+
{
150+
insertExtraDefines(code, preprocessOptions.extraDefines);
151+
}
152+
if (preprocessOptions.includeFinder != nullptr)
153+
{
154+
tcpp::StringInputStream codeIs = tcpp::StringInputStream(code);
155+
tcpp::Lexer lexer(codeIs);
156+
tcpp::Preprocessor proc(
157+
lexer,
158+
[](auto errorInfo) {
159+
160+
},
161+
[&](auto path, auto isSystemPath) {
162+
return getInputStreamInclude(
163+
preprocessOptions.includeFinder, m_system.get(), preprocessOptions.maxSelfInclusionCount,
164+
path.c_str(), !isSystemPath
165+
);
166+
}
167+
);
168+
169+
return proc.Process();
170+
}
171+
else
172+
{
173+
return code;
174+
}
175+
}
176+
99177
core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* code, const IShaderCompiler::SCompilerOptions& options) const
100178
{
101179
auto hlslOptions = option_cast(options);
@@ -105,8 +183,8 @@ core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* cod
105183
hlslOptions.preprocessorOptions.logger.log("code is nullptr", system::ILogger::ELL_ERROR);
106184
return nullptr;
107185
}
108-
109-
auto newCode = std::string(code);//preprocessShader(code, hlslOptions.stage, hlslOptions.preprocessorOptions);
186+
187+
auto newCode = preprocessShader(code, hlslOptions.stage, hlslOptions.preprocessorOptions);
110188

111189
// Suffix is the shader model version
112190
std::wstring targetProfile(L"XX_6_2");
@@ -156,7 +234,7 @@ core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* cod
156234
const uint32_t nonDebugArgs = 5;
157235
const uint32_t allArgs = nonDebugArgs + 2;
158236

159-
DxcCompilationResult compileResult = dxcCompile(newCode, &arguments[0], hlslOptions.genDebugInfo ? allArgs : nonDebugArgs, hlslOptions);
237+
auto compileResult = dxcCompile(newCode, &arguments[0], hlslOptions.genDebugInfo ? allArgs : nonDebugArgs, hlslOptions);
160238

161239
if (!compileResult.objectBlob)
162240
{

0 commit comments

Comments
 (0)