Merge branch 'master' into more_fft_utils

Author: Fletterio

3rdparty/dxc/dxc

@@ -25,7 +25,7 @@ class ICPUComputePipeline : public ICPUPipeline<IPipeline<ICPUPipelineLayout>,1>
         {
             if (!params.layout)
                 return nullptr;
-            auto retval = new ICPUComputePipeline(core::smart_refctd_ptr<ICPUPipelineLayout>(params.layout));
+            auto retval = new ICPUComputePipeline(core::smart_refctd_ptr<const ICPUPipelineLayout>(params.layout));
             if (!retval->setSpecInfo(params.shader))
             {
                 retval->drop();
@@ -48,7 +48,7 @@ class ICPUComputePipeline : public ICPUPipeline<IPipeline<ICPUPipelineLayout>,1>
         using base_t::base_t;
         virtual ~ICPUComputePipeline() = default;
 
-        base_t* clone_impl(core::smart_refctd_ptr<ICPUPipelineLayout>&& layout) const override
+        base_t* clone_impl(core::smart_refctd_ptr<const ICPUPipelineLayout>&& layout) const override 
         {
             return new ICPUComputePipeline(std::move(layout));
         }
@@ -57,7 +57,7 @@ class ICPUComputePipeline : public ICPUPipeline<IPipeline<ICPUPipelineLayout>,1>
         {
             if (ix!=0)
                 return m_stages[0].shader.get();
-            return m_layout.get();
+            return const_cast<ICPUPipelineLayout*>(m_layout.get());
         }
 
         inline int8_t stageToIndex(const ICPUShader::E_SHADER_STAGE stage) const override

examples_tests

@@ -65,7 +65,7 @@ class ICPUGraphicsPipeline final : public ICPUPipeline<IGraphicsPipeline<ICPUPip
 		using base_t::base_t;
         ~ICPUGraphicsPipeline() = default;
 
-		base_t* clone_impl(core::smart_refctd_ptr<ICPUPipelineLayout>&& layout) const override
+		base_t* clone_impl(core::smart_refctd_ptr<const ICPUPipelineLayout>&& layout) const override
 		{
 			std::array<ICPUShader::SSpecInfo,GRAPHICS_SHADER_STAGE_COUNT> _shaders;
 			for (auto i=0; i<GRAPHICS_SHADER_STAGE_COUNT; i++)
@@ -80,7 +80,7 @@ class ICPUGraphicsPipeline final : public ICPUPipeline<IGraphicsPipeline<ICPUPip
 		inline IAsset* getDependant_impl(const size_t ix) override
 		{
 			if (ix==0)
-				return m_layout.get();
+				return const_cast<ICPUPipelineLayout*>(m_layout.get());
 			if (ix==1)
 				return m_renderpass.get();
 			size_t stageCount = 0;

include/nbl/asset/ICPUComputePipeline.h

@@ -51,11 +51,11 @@ class ICPUPipeline : public IAsset, public PipelineNonAssetBase
         ICPUPipelineLayout* getLayout() 
         {
             assert(isMutable());
-            return PipelineNonAssetBase::m_layout.get(); 
+            return const_cast<ICPUPipelineLayout*>(PipelineNonAssetBase::m_layout.get());
         }
         const ICPUPipelineLayout* getLayout() const { return PipelineNonAssetBase::m_layout.get(); }
 
-        inline void setLayout(core::smart_refctd_ptr<ICPUPipelineLayout>&& _layout)
+        inline void setLayout(core::smart_refctd_ptr<const ICPUPipelineLayout>&& _layout)
         {
             assert(isMutable());
             PipelineNonAssetBase::m_layout = std::move(_layout);
@@ -117,7 +117,7 @@ class ICPUPipeline : public IAsset, public PipelineNonAssetBase
         using PipelineNonAssetBase::PipelineNonAssetBase;
         virtual ~ICPUPipeline() = default;
 
-        virtual this_t* clone_impl(core::smart_refctd_ptr<ICPUPipelineLayout>&& layout) const = 0;
+        virtual this_t* clone_impl(core::smart_refctd_ptr<const ICPUPipelineLayout>&& layout) const = 0;
         virtual int8_t stageToIndex(const ICPUShader::E_SHADER_STAGE stage) const = 0;
 
         struct ShaderStage {

include/nbl/asset/ICPUGraphicsPipeline.h

@@ -155,7 +155,7 @@ class IGraphicsPipeline : public IPipeline<PipelineLayoutType>, public IGraphics
 
     protected:
         explicit IGraphicsPipeline(const SCreationParams& _params) :
-            IPipeline<PipelineLayoutType>(core::smart_refctd_ptr<PipelineLayoutType>(_params.layout)),
+            IPipeline<PipelineLayoutType>(core::smart_refctd_ptr<const PipelineLayoutType>(_params.layout)),
             m_params(_params.cached), m_renderpass(core::smart_refctd_ptr<renderpass_t>(_params.renderpass)) {}
 
         SCachedCreationParams m_params;

include/nbl/asset/ICPUPipeline.h

@@ -35,7 +35,7 @@ class IPipeline
 		struct SCreationParams
 		{
 			public:
-				PipelineLayout* layout = nullptr;
+				const PipelineLayout* layout = nullptr;
 
 			protected:
 				// This is not public to make sure that different pipelines only get the enums they support
@@ -107,9 +107,9 @@ class IPipeline
 		inline const PipelineLayout* getLayout() const {return m_layout.get();}
 
 	protected:
-		inline IPipeline(core::smart_refctd_ptr<PipelineLayout>&& _layout) : m_layout(std::move(_layout)) {}
+		inline IPipeline(core::smart_refctd_ptr<const PipelineLayout>&& _layout) : m_layout(std::move(_layout)) {}
 
-		core::smart_refctd_ptr<PipelineLayout> m_layout;
+		core::smart_refctd_ptr<const PipelineLayout> m_layout;
 };
 
 }

include/nbl/asset/IGraphicsPipeline.h

@@ -4,12 +4,14 @@
 #ifndef _NBL_ASSET_I_PIPELINE_LAYOUT_H_INCLUDED_
 #define _NBL_ASSET_I_PIPELINE_LAYOUT_H_INCLUDED_
 
+#include "nbl/macros.h"
+#include "nbl/core/declarations.h"
 
 #include <algorithm>
 #include <array>
 
-#include "nbl/macros.h"
-#include "nbl/core/declarations.h"
+#include "nbl/asset/IDescriptorSetLayout.h"
+#include "nbl/builtin/hlsl/binding_info.hlsl"
 
 
 namespace nbl::asset
@@ -21,7 +23,7 @@ namespace nbl::asset
     however they serve as a fast path with regard to data upload from the
     CPU and data access from the GPU. 
     
-    Note that IrrlichtBaW limits push constant size to 128 bytes.
+    Note that Nabla limits push constant size to 128 bytes.
 
     Push Constants are an alternative to an UBO where it performs really poorly,
     mostly very small and very frequent updates. Examples of which are:
@@ -140,6 +142,54 @@ class IPipelineLayout
             return static_cast<int32_t>(i)-1;
         }
 
+        // utility function, if you compile shaders for specific layouts, not create layouts given shaders
+        struct SBindingKey
+        {
+            using type_bitset_t = std::bitset<static_cast<size_t>(IDescriptor::E_TYPE::ET_COUNT)>;
+
+            hlsl::SBindingInfo binding = {};
+            core::bitflag<IShader::E_SHADER_STAGE> requiredStages = IShader::E_SHADER_STAGE::ESS_UNKNOWN;
+            // could have just initialized with `~type_bitset_t()` in C++23
+            type_bitset_t allowedTypes = type_bitset_t((0x1u<<static_cast<size_t>(IDescriptor::E_TYPE::ET_COUNT))-1);
+        };
+        // TODO: add constraints for stage and creation flags, or just return the storage index & redirect?
+        core::string getBindingInfoForHLSL(const SBindingKey& key) const
+        {
+            if (key.binding.set>=DESCRIPTOR_SET_COUNT)
+                return "#error \"IPipelineLayout::SBindingKey::binding::set out of range!\"";
+            const auto* layout = m_descSetLayouts[key.binding.set].get();
+            if (!layout)
+                return "#error \"IPipelineLayout::SBindingKey::binding::set layout is nullptr!\"";
+            //
+            using redirect_t = IDescriptorSetLayoutBase::CBindingRedirect;
+            using storage_range_index_t = redirect_t::storage_range_index_t;
+            const redirect_t* redirect;
+            storage_range_index_t found;
+            {
+                const redirect_t::binding_number_t binding(key.binding.binding);
+                for (auto t=0u; t<static_cast<size_t>(IDescriptor::E_TYPE::ET_COUNT); t++)
+                if (key.allowedTypes.test(t))
+                {
+                    redirect = &layout->getDescriptorRedirect(static_cast<IDescriptor::E_TYPE>(t));
+                    found = redirect->findBindingStorageIndex(binding);
+                    if (found)
+                        break;
+                }
+                if (!found && key.allowedTypes.test(static_cast<size_t>(IDescriptor::E_TYPE::ET_SAMPLER)))
+                {
+                    redirect = &layout->getImmutableSamplerRedirect();
+                    found = redirect->findBindingStorageIndex(binding);
+                }
+                if (!found)
+                    return "#error \"Could not find `IPipelineLayout::SBindingKey::binding::binding` in `IPipelineLayout::SBindingKey::binding::set`'s layout!\"";
+            }
+            if (!redirect->getStageFlags(found).hasFlags(key.requiredStages))
+                return "#error \"Binding found in the layout doesn't have all the `IPipelineLayout::SBindingKey::binding::requiredStages` flags!\"";
+            const auto count = redirect->getCount(found);
+            assert(count); // this layout should have never passed validation
+            return "::nbl::hlsl::ConstevalBindingInfo<"+std::to_string(key.binding.set)+","+std::to_string(key.binding.binding)+","+std::to_string(count)+">";
+        }
+
     protected:
         IPipelineLayout(
             const std::span<const asset::SPushConstantRange> _pcRanges,

include/nbl/asset/IPipeline.h

@@ -33,7 +33,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 				{
 					system::path absolutePath = {};
 					std::string contents = {};
-					std::array<uint64_t, 4> hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future
+					core::blake3_hash_t hash = {}; // TODO: we're not yet using IFile::getPrecomputedHash(), so for builtins we can maybe use that in the future
 					// Could be used in the future for early rejection of cache hit
 					//nbl::system::IFileBase::time_point_t lastWriteTime = {};
 
@@ -183,9 +183,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 
 			public:
 				// Used to check compatibility of Caches before reading
-				constexpr static inline std::string_view VERSION = "1.0.0";
+				constexpr static inline std::string_view VERSION = "1.1.0";
 
-				using hash_t = std::array<uint64_t,4>;
 				static auto const SHADER_BUFFER_SIZE_BYTES = sizeof(uint64_t) / sizeof(uint8_t); // It's obviously 8
 
 				struct SEntry
@@ -196,11 +195,9 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 					{
 						public:
 							// Perf note: hashing while preprocessor lexing is likely to be slower than just hashing the whole array like this 
-							inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, const std::string_view& _contents, bool _standardInclude, std::array<uint64_t, 4> _hash) :
-								requestingSourceDir(_requestingSourceDir), identifier(_identifier), contents(_contents), standardInclude(_standardInclude), hash(_hash)
-							{
-								assert(!_contents.empty());
-							}
+							inline SPreprocessingDependency(const system::path& _requestingSourceDir, const std::string_view& _identifier, bool _standardInclude, core::blake3_hash_t _hash) :
+								requestingSourceDir(_requestingSourceDir), identifier(_identifier), standardInclude(_standardInclude), hash(_hash)
+							{}
 
 							inline SPreprocessingDependency(SPreprocessingDependency&) = default;
 							inline SPreprocessingDependency& operator=(SPreprocessingDependency&) = delete;
@@ -218,11 +215,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 							// path or identifier
 							system::path requestingSourceDir = "";
 							std::string identifier = "";
-							// file contents
-							// TODO: change to `core::vector<uint8_t>` a compressed blob of LZMA, and store all contents together in the `SEntry`
-							std::string contents = ""; 
 							// hash of the contents - used to check against a found_t
-							std::array<uint64_t, 4> hash = {};
+							core::blake3_hash_t hash = {};
 							// If true, then `getIncludeStandard` was used to find, otherwise `getIncludeRelative`
 							bool standardInclude = false;
 					};
@@ -248,6 +242,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 						private:
 							friend class SCompilerArgs;
 							friend class SEntry;
+							friend class CCache;
 							friend void to_json(nlohmann::json&, const SPreprocessorArgs&);
 							friend void from_json(const nlohmann::json&, SPreprocessorArgs&);
 
@@ -271,7 +266,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 							std::string sourceIdentifier;
 							std::vector<SMacroDefinition> extraDefines;
 					};
-					// TODO: SPreprocessorArgs could just be folded into `SCompilerArgs` to have less classes and operators
+					// TODO: SPreprocessorArgs could just be folded into `SCompilerArgs` to have less classes and decompressShader
 					struct SCompilerArgs final
 					{
 						public:
@@ -290,6 +285,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 
 						private:
 							friend class SEntry;
+							friend class CCache;
 							friend void to_json(nlohmann::json&, const SCompilerArgs&);
 							friend void from_json(const nlohmann::json&, SCompilerArgs&);
 
@@ -351,33 +347,40 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 
 						// Now add the mainFileContents and produce both lookup and early equality rejection hashes
 						hashable.insert(hashable.end(), mainFileContents.begin(), mainFileContents.end());
-						hash = nbl::core::XXHash_256(hashable.data(), hashable.size());
-						lookupHash = hash[0];
-						for (auto i = 1u; i < 4; i++) {
-							core::hash_combine<uint64_t>(lookupHash, hash[i]);
-						}
+
+						core::blake3_hasher hasher;
+						hasher.update(hashable.data(), hashable.size());
+						hash = static_cast<core::blake3_hash_t>(hasher);
+						lookupHash = std::hash<core::blake3_hash_t>{}(hash);
 					}
 
 					// Needed to get the vector deserialization automatically
 					inline SEntry() {}
 
 					// Making the copy constructor deep-copy everything but the shader 
-					inline SEntry(const SEntry& other) 
-						: mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash), lookupHash(other.lookupHash), 
-						  dependencies(other.dependencies), cpuShader(other.cpuShader) {}
+					inline SEntry(const SEntry& other)
+						: mainFileContents(other.mainFileContents), compilerArgs(other.compilerArgs), hash(other.hash),
+						lookupHash(other.lookupHash), dependencies(other.dependencies), spirv(other.spirv),
+						uncompressedContentHash(other.uncompressedContentHash), uncompressedSize(other.uncompressedSize) {}
 
 					inline SEntry& operator=(SEntry& other) = delete;
 					inline SEntry(SEntry&& other) = default;
 					// Used for late initialization while looking up a cache, so as not to always initialize an entry even if caching was not requested
 					inline SEntry& operator=(SEntry&& other) = default;
 
+					bool setContent(const asset::ICPUBuffer* uncompressedSpirvBuffer, dependency_container_t&& dependencies);
+
+					core::smart_refctd_ptr<ICPUShader> decompressShader() const;
+
 					// TODO: make some of these private
 					std::string mainFileContents;
 					SCompilerArgs compilerArgs;
-					std::array<uint64_t,4> hash;
+					core::blake3_hash_t hash;
 					size_t lookupHash;
 					dependency_container_t dependencies;
-					core::smart_refctd_ptr<asset::ICPUShader> cpuShader;
+					core::smart_refctd_ptr<asset::ICPUBuffer> spirv;
+					core::blake3_hash_t uncompressedContentHash;
+					size_t uncompressedSize;
 				};
 
 				inline void insert(SEntry&& entry)
@@ -429,52 +432,13 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 
 				};
 
-				using EntrySet = core::unordered_multiset<SEntry, Hash, KeyEqual>;
+				using EntrySet = core::unordered_set<SEntry, Hash, KeyEqual>;
 				EntrySet m_container;
 
 				NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const;
 		};
 
-		inline core::smart_refctd_ptr<ICPUShader> compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const
-		{
-			CCache::SEntry entry;
-			std::vector<CCache::SEntry::SPreprocessingDependency> dependencies;
-			if (options.readCache || options.writeCache)
-				entry = std::move(CCache::SEntry(code, options));
-			
-			if (options.readCache)
-			{
-				auto found = options.readCache->find_impl(entry, options.preprocessorOptions.includeFinder);
-				if (found != options.readCache->m_container.end())
-				{
-					if (options.writeCache)
-					{
-						CCache::SEntry writeEntry = *found;
-						options.writeCache->insert(std::move(writeEntry));
-					}
-					return found->cpuShader;
-				}
-			}
-
-			auto retVal = compileToSPIRV_impl(code, options, options.writeCache ? &dependencies : nullptr);
-
-			if (!retVal)
-				return nullptr;
-
-			// compute the SPIR-V shader content hash
-			{
-				auto backingBuffer = retVal->getContent();
-				const_cast<ICPUBuffer*>(backingBuffer)->setContentHash(backingBuffer->computeContentHash());
-			}
-
-			if (options.writeCache)
-			{
-				entry.dependencies = std::move(dependencies);
-				entry.cpuShader = retVal;
-				options.writeCache->insert(std::move(entry));
-			}
-			return retVal;
-		}
+		core::smart_refctd_ptr<ICPUShader> compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const;
 
 		inline core::smart_refctd_ptr<ICPUShader> compileToSPIRV(const char* code, const SCompilerOptions& options) const
 		{

include/nbl/asset/IPipelineLayout.h

@@ -0,0 +1,33 @@
+// Copyright (C) 2024 - DevSH Graphics Programming Sp. z O.O.
+// This file is part of the "Nabla Engine".
+// For conditions of distribution and use, see copyright notice in nabla.h
+#ifndef _NBL_BUILTIN_HLSL_BINDING_INFO_INCLUDED_
+#define _NBL_BUILTIN_HLSL_BINDING_INFO_INCLUDED_
+
+#include "nbl/builtin/hlsl/cpp_compat.hlsl"
+
+namespace nbl
+{
+namespace hlsl
+{
+
+template<uint32_t set, uint32_t ix, uint32_t count=1>
+struct ConstevalBindingInfo
+{
+	NBL_CONSTEXPR_STATIC_INLINE uint32_t Set = set;
+	NBL_CONSTEXPR_STATIC_INLINE uint32_t Index = ix;
+	NBL_CONSTEXPR_STATIC_INLINE uint32_t Count = count;
+};
+
+// used for descriptor set layout lookups
+struct SBindingInfo
+{
+	//! binding index for a given resource
+	uint32_t binding : 29;
+	//! descriptor set index for a resource
+	uint32_t set : 3;
+};
+
+}
+}
+#endif