Skip to content

Commit 787b887

Browse files
committed
Shader Compiler Tool
1 parent b60490b commit 787b887

File tree

4 files changed

+210
-65
lines changed

4 files changed

+210
-65
lines changed

.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,7 @@ android-sample/bin/*
2929
preprocessed.hlsl
3030
compiled.spv
3131
.vs/*
32+
tools/ndt/__main__.py
33+
tools/ndt/.vscode/settings.json
34+
tools/ndt/.vscode/launch.json
35+
tools/ndt/.profiles/0.json

include/nbl/asset/utils/CHLSLCompiler.h

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@
77

88
#include "nbl/asset/utils/ISPIRVOptimizer.h"
99
#include "nbl/asset/utils/IShaderCompiler.h"
10+
#include <wrl.h>
11+
#include <combaseapi.h>
12+
#include <sstream>
13+
#include <dxc/dxcapi.h>
14+
15+
1016

1117
#ifdef _NBL_PLATFORM_WINDOWS_
1218

@@ -21,6 +27,19 @@ namespace nbl::asset
2127
class NBL_API2 CHLSLCompiler final : public IShaderCompiler
2228
{
2329
public:
30+
31+
struct DxcCompilationResult
32+
{
33+
Microsoft::WRL::ComPtr<IDxcBlobEncoding> errorMessages;
34+
Microsoft::WRL::ComPtr<IDxcBlob> objectBlob;
35+
Microsoft::WRL::ComPtr<IDxcResult> compileResult;
36+
37+
std::string GetErrorMessagesString()
38+
{
39+
return std::string(reinterpret_cast<char*>(errorMessages->GetBufferPointer()), errorMessages->GetBufferSize());
40+
}
41+
};
42+
2443
IShader::E_CONTENT_TYPE getCodeContentType() const override { return IShader::E_CONTENT_TYPE::ECT_HLSL; };
2544

2645
CHLSLCompiler(core::smart_refctd_ptr<system::ISystem>&& system);
@@ -50,7 +69,10 @@ class NBL_API2 CHLSLCompiler final : public IShaderCompiler
5069
std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, std::vector<std::string>& dxc_compile_flags_override, const SPreprocessorOptions& preprocessOptions) const;
5170

5271
void insertIntoStart(std::string& code, std::ostringstream&& ins) const override;
72+
73+
DxcCompilationResult dxcCompile(std::string& source, LPCWSTR* args, uint32_t argCount, const CHLSLCompiler::SOptions& options) const;
5374
protected:
75+
5476

5577
// This can't be a unique_ptr due to it being an undefined type
5678
// when Nabla is used as a lib

src/nbl/asset/utils/CHLSLCompiler.cpp

Lines changed: 4 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,13 @@
33
// For conditions of distribution and use, see copyright notice in nabla.h
44
#include "nbl/asset/utils/CHLSLCompiler.h"
55
#include "nbl/asset/utils/shadercUtils.h"
6-
// TODO: review
76
#ifdef NBL_EMBED_BUILTIN_RESOURCES
87
#include "nbl/builtin/CArchive.h"
98
#include "spirv/builtin/CArchive.h"
109
#endif // NBL_EMBED_BUILTIN_RESOURCES
1110

1211
#ifdef _NBL_PLATFORM_WINDOWS_
1312

14-
#include <wrl.h>
15-
#include <combaseapi.h>
16-
17-
#include <dxc/dxcapi.h>
18-
19-
#include <sstream>
2013
#include <regex>
2114
#include <iterator>
2215
#include <codecvt>
@@ -60,20 +53,7 @@ CHLSLCompiler::~CHLSLCompiler()
6053
delete m_dxcCompilerTypes;
6154
}
6255

63-
64-
struct DxcCompilationResult
65-
{
66-
ComPtr<IDxcBlobEncoding> errorMessages;
67-
ComPtr<IDxcBlob> objectBlob;
68-
ComPtr<IDxcResult> compileResult;
69-
70-
std::string GetErrorMessagesString()
71-
{
72-
return std::string(reinterpret_cast<char*>(errorMessages->GetBufferPointer()), errorMessages->GetBufferSize());
73-
}
74-
};
75-
76-
DxcCompilationResult dxcCompile(const CHLSLCompiler* compiler, nbl::asset::impl::DXC* dxc, std::string& source, LPCWSTR* args, uint32_t argCount, const CHLSLCompiler::SOptions& options)
56+
CHLSLCompiler::DxcCompilationResult CHLSLCompiler::dxcCompile(std::string& source, LPCWSTR* args, uint32_t argCount, const CHLSLCompiler::SOptions& options) const
7757
{
7858
// Append Commandline options into source only if debugInfoFlags will emit source
7959
auto sourceEmittingFlags =
@@ -93,9 +73,9 @@ DxcCompilationResult dxcCompile(const CHLSLCompiler* compiler, nbl::asset::impl:
9373
}
9474

9575
insertion << "\n";
96-
compiler->insertIntoStart(source, std::move(insertion));
76+
insertIntoStart(source, std::move(insertion));
9777
}
98-
78+
nbl::asset::impl::DXC* dxc = m_dxcCompilerTypes;
9979
ComPtr<IDxcBlobEncoding> src;
10080
auto res = dxc->m_dxcUtils->CreateBlob(reinterpret_cast<const void*>(source.data()), source.size(), CP_UTF8, &src);
10181
assert(SUCCEEDED(res));
@@ -317,9 +297,7 @@ core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* cod
317297
arguments.push_back(L"-fspv-debug=vulkan-with-source");
318298
}
319299

320-
auto compileResult = dxcCompile(
321-
this,
322-
m_dxcCompilerTypes,
300+
auto compileResult = dxcCompile(
323301
newCode,
324302
arguments.data(),
325303
arguments.size(),

tools/ndt/main.cpp

Lines changed: 180 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -7,54 +7,195 @@
77

88
#include <iostream>
99
#include <cstdlib>
10+
#include <string>
11+
#include <fstream>
12+
#include <streambuf>
1013

14+
using namespace nbl;
15+
using namespace nbl::system;
16+
using namespace nbl::core;
17+
using namespace nbl::asset;
1118

1219

13-
class ShaderCompiler final : public system::IApplicationFramework
14-
{
15-
using base_t = system::IApplicationFramework;
16-
17-
public:
18-
using base_t::base_t;
19-
20-
bool onAppInitialized(smart_refctd_ptr<ISystem>&& system) override
21-
{
22-
if (!base_t::onAppInitialized(std::move(system)))
23-
return false;
24-
25-
auto argc = argv.size();
26-
27-
core::vector<std::string> arguments(argv + 1, argv + argc);
28-
29-
for (auto i=0; i<argc; i++)
30-
{
31-
if (argv[i] == "-no-nbl-builtins")
32-
{
33-
no_nbl_builtins = true;
34-
break;
35-
}
36-
}
3720

38-
std::string command = "dxc.exe";
39-
for (std::string arg : arguments)
40-
{
41-
command.append(" ").append(arg);
42-
}
4321

44-
int execute = std::system(command.c_str());
4522

46-
std::cout << "-no-nbl-builtins - " << no_nbl_builtins;
4723

4824

49-
return true;
50-
}
51-
52-
53-
void workLoopBody() override {}
25+
class ShaderCompiler final : public system::IApplicationFramework
26+
{
27+
using base_t = system::IApplicationFramework;
5428

55-
bool keepRunning() override { return false; }
29+
public:
30+
using base_t::base_t;
31+
32+
bool onAppInitialized(smart_refctd_ptr<ISystem>&& system) override
33+
{
34+
if (system)
35+
m_system = std::move(system);
36+
else
37+
m_system = system::IApplicationFramework::createSystem();
38+
39+
m_logger = make_smart_refctd_ptr<CStdoutLogger>();
40+
41+
auto argc = argv.size();
42+
43+
// expect the first argument to be
44+
// nsc.exe
45+
// second the filename of a shader to compile
46+
if (argc < 2) {
47+
m_logger->log("Insufficient arguments.", ILogger::ELL_ERROR);
48+
return false;
49+
}
50+
51+
m_arguments = std::vector<std::string>(argv.begin() + 1, argv.end()); // turn argv into vector for convenience
52+
std::string file_to_compile = m_arguments[0];
53+
54+
if (!m_system->exists(file_to_compile, IFileBase::ECF_READ)) {
55+
m_logger->log("Incorrect arguments. Expecting second argument to be filename of the shader intended to compile.", ILogger::ELL_ERROR);
56+
return false;
57+
}
58+
std::string output_filepath;
59+
for (auto i = 1; i < argc; i++)
60+
{
61+
if (argv[i] == "-no-nbl-builtins")
62+
{
63+
no_nbl_builtins = true;
64+
}
65+
else if (argv[i] == "-Fo")
66+
{
67+
if (i + 1 < argc) {
68+
i++;
69+
output_filepath = argv[i];
70+
m_logger->log("Saving compiled shader code to " + output_filepath);
71+
}
72+
else {
73+
m_logger->log("Incorrect arguments. Expecting filename after -Fo.", ILogger::ELL_ERROR);
74+
}
75+
}
76+
}
77+
string shader_code = open_shader_file(file_to_compile);
78+
auto compilation_result = compile_shader(shader_code, file_to_compile);
79+
80+
// writie compiled shader to file as bytes
81+
std::fstream output_file(output_filepath, std::ios::out | std::ios::binary);
82+
output_file.write((const char*)compilation_result.objectBlob->GetBufferPointer(), compilation_result.objectBlob->GetBufferSize());
83+
output_file.close();
84+
/*std::string command = "dxc.exe";
85+
for (std::string arg : arguments)
86+
{
87+
command.append(" ").append(arg);
88+
}
89+
90+
int execute = std::system(command.c_str());*/
91+
92+
//std::cout << "-no-nbl-builtins - " << no_nbl_builtins;
93+
94+
95+
return true;
96+
}
97+
98+
99+
void workLoopBody() override {}
100+
101+
bool keepRunning() override { return false; }
56102

57103

58104
private:
59-
bool no_nbl_builtins{false};
60-
};
105+
106+
CHLSLCompiler::DxcCompilationResult compile_shader(std::string& shader_code, std::string_view sourceIdentifier) {
107+
constexpr uint32_t WorkgroupSize = 256;
108+
constexpr uint32_t WorkgroupCount = 2048;
109+
const string WorkgroupSizeAsStr = std::to_string(WorkgroupSize);
110+
const IShaderCompiler::SPreprocessorOptions::SMacroDefinition WorkgroupSizeDefine = { "WORKGROUP_SIZE",WorkgroupSizeAsStr };
111+
112+
smart_refctd_ptr<CHLSLCompiler> hlslcompiler = make_smart_refctd_ptr<CHLSLCompiler>(std::move(m_system));
113+
114+
CHLSLCompiler::SOptions options = {};
115+
options.stage = asset::IShader::E_SHADER_STAGE::ESS_UNKNOWN; // probably not needed, requires guessing -T target profile
116+
// want as much debug as possible
117+
options.debugInfoFlags = IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_LINE_BIT;
118+
// this lets you source-level debug/step shaders in renderdoc
119+
//if (physDev->getLimits().shaderNonSemanticInfo)
120+
//options.debugInfoFlags |= IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_NON_SEMANTIC_BIT;
121+
// if you don't set the logger and source identifier you'll have no meaningful errors
122+
options.preprocessorOptions.sourceIdentifier = sourceIdentifier;
123+
options.preprocessorOptions.logger = m_logger.get();
124+
options.preprocessorOptions.extraDefines = { &WorkgroupSizeDefine,&WorkgroupSizeDefine + 1 };
125+
126+
std::vector<std::string> dxc_compile_flags_from_pragma = {};
127+
auto preprocessed_shader_code = hlslcompiler->preprocessShader(std::move(shader_code), options.stage, dxc_compile_flags_from_pragma, options.preprocessorOptions);
128+
129+
// override arguments from command line to ones listed in pragma
130+
if (dxc_compile_flags_from_pragma.size())
131+
m_arguments = dxc_compile_flags_from_pragma;
132+
133+
add_required_arguments_if_not_present();
134+
135+
//convert string arguments to wstring arguments
136+
int arg_size = m_arguments.size() - 1; // skip input file argument
137+
LPCWSTR* arguments = new LPCWSTR[arg_size]; //array of pointers
138+
std::wstring* arguments_wdata = new std::wstring[arg_size]; // array of data, prevents deallocation before shader compilation
139+
for (size_t i = 0; i < arg_size; i++) {
140+
arguments_wdata[i] = std::wstring(m_arguments[i + 1].begin(), m_arguments[i + 1].end());
141+
arguments[i] = arguments_wdata[i].c_str();
142+
}
143+
144+
auto compileResult = hlslcompiler->dxcCompile(preprocessed_shader_code, arguments, arg_size, options);
145+
delete[] arguments;
146+
delete[] arguments_wdata;
147+
return compileResult;
148+
}
149+
150+
151+
void add_required_arguments_if_not_present() {
152+
constexpr int required_arg_size = 8;
153+
std::string required_arguments[required_arg_size] = {
154+
"-spirv",
155+
"-Zpr", // Packs matrices in row-major order by default
156+
"-enable-16bit-types",
157+
"-fvk-use-scalar-layout",
158+
"-Wno-c++11-extensions",
159+
"-Wno-c++1z-extensions",
160+
"-Wno-gnu-static-float-init",
161+
"-fspv-target-env=vulkan1.3"
162+
};
163+
bool found_arg_flags[required_arg_size]{};
164+
int argc = m_arguments.size();
165+
for (int i = 0; i < argc; i++)
166+
{
167+
for (int j = 0; j < required_arg_size; j++)
168+
{
169+
if (m_arguments[i] == required_arguments[j]) {
170+
found_arg_flags[j] = true;
171+
break;
172+
}
173+
}
174+
}
175+
for (int j = 0; j < required_arg_size; j++)
176+
{
177+
if (!found_arg_flags[j]) {
178+
m_logger->log("Required compile flag not found " + required_arguments[j] +". This flag will be force enabled as it is required by Nabla.", ILogger::ELL_WARNING);
179+
m_arguments.push_back(required_arguments[j]);
180+
}
181+
}
182+
183+
}
184+
185+
186+
std::string open_shader_file(std::string& filepath) {
187+
std::ifstream stream(filepath);
188+
std::string str((std::istreambuf_iterator<char>(stream)),
189+
std::istreambuf_iterator<char>());
190+
return str;
191+
}
192+
193+
194+
bool no_nbl_builtins{ false };
195+
smart_refctd_ptr<ISystem> m_system;
196+
smart_refctd_ptr<CStdoutLogger> m_logger;
197+
std::vector<std::string> m_arguments;
198+
199+
};
200+
201+
NBL_MAIN_FUNC(ShaderCompiler)

0 commit comments

Comments
 (0)