Add setRayTracingStackSize validation

Author: kevyuu

include/nbl/video/IGPUCommandBuffer.h

@@ -720,6 +720,7 @@ class NBL_API2 IGPUCommandBuffer : public IBackendObject
             m_boundGraphicsPipeline= nullptr;
             m_boundComputePipeline= nullptr;
             m_boundRayTracingPipeline= nullptr;
+            m_haveRtPipelineStackSize = false;
 
             m_commandList.head = nullptr;
             m_commandList.tail = nullptr;
@@ -736,6 +737,7 @@ class NBL_API2 IGPUCommandBuffer : public IBackendObject
             m_boundGraphicsPipeline= nullptr;
             m_boundComputePipeline= nullptr;
             m_boundRayTracingPipeline= nullptr;
+            m_haveRtPipelineStackSize = false;
             releaseResourcesBackToPool_impl();
         }
 
@@ -878,6 +880,7 @@ class NBL_API2 IGPUCommandBuffer : public IBackendObject
         uint64_t m_resetCheckedStamp;
         STATE m_state = STATE::INITIAL;
         bool m_noCommands = true;
+        bool m_haveRtPipelineStackSize = false;
         // only useful while recording
         SInheritanceInfo m_cachedInheritanceInfo;
         core::bitflag<USAGE> m_recordingFlags = USAGE::NONE;

src/nbl/video/IGPUCommandBuffer.cpp

@@ -1890,6 +1890,9 @@ bool IGPUCommandBuffer::resolveImage(const IGPUImage* const srcImage, const IGPU
 
 bool IGPUCommandBuffer::setRayTracingPipelineStackSize(uint32_t pipelineStackSize)
 {
+    if (!checkStateBeforeRecording(queue_flags_t::COMPUTE_BIT,RENDERPASS_SCOPE::OUTSIDE))
+        return false;
+    m_haveRtPipelineStackSize = true;
     return setRayTracingPipelineStackSize_impl(pipelineStackSize);
 }
 
@@ -1937,6 +1940,13 @@ bool IGPUCommandBuffer::traceRays(
         return false;
     }
 
+    // https://docs.vulkan.org/spec/latest/chapters/raytracing.html#VUID-vkCmdTraceRaysKHR-None-09458
+    if (m_boundRayTracingPipeline->getCachedCreationParams().dynamicStackSize && !m_haveRtPipelineStackSize)
+    {
+        NBL_LOG_ERROR("no setRayTracingPipelineStackSize command submitted before traceRays command with dynamic stack size pipeline!");
+        return false;
+    }
+
     if (!m_cmdpool->m_commandListPool.emplace<IGPUCommandPool::CTraceRaysCmd>(m_commandList, 
         core::smart_refctd_ptr<const IGPUBuffer>(raygenGroupRange.buffer),
         core::smart_refctd_ptr<const IGPUBuffer>(missGroupsRange.buffer),
@@ -1984,6 +1994,13 @@ bool IGPUCommandBuffer::traceRaysIndirect(
         return false;
     }
 
+    // https://docs.vulkan.org/spec/latest/chapters/raytracing.html#VUID-vkCmdTraceRaysIndirect2KHR-None-09458
+    if (m_boundRayTracingPipeline->getCachedCreationParams().dynamicStackSize && !m_haveRtPipelineStackSize)
+    {
+        NBL_LOG_ERROR("no setRayTracingPipelineStackSize command submitted before traceRays command with dynamic stack size pipeline!");
+        return false;
+    }
+
     if (!m_cmdpool->m_commandListPool.emplace<IGPUCommandPool::CTraceRaysIndirectCmd>(m_commandList, 
         core::smart_refctd_ptr<const IGPUBuffer>(raygenGroupRange.buffer),
         core::smart_refctd_ptr<const IGPUBuffer>(missGroupsRange.buffer),