From 6218164f79c7deb8a00b1093b3879313203f965e Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Tue, 4 Jun 2024 12:17:06 -0700 Subject: [PATCH 01/43] Adding the begin frame message for later xtrace post processing. --- filament/backend/src/vulkan/VulkanDriver.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 60669cc3398d..7eed4d867cdc 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -397,7 +397,10 @@ void VulkanDriver::collectGarbage() { } void VulkanDriver::beginFrame(int64_t monotonic_clock_ns, int64_t refreshIntervalNs, uint32_t frameId) { + FVK_SYSTRACE_CONTEXT(); + FVK_SYSTRACE_START("beginFrame"); // Do nothing. + FVK_SYSTRACE_END(); } void VulkanDriver::setFrameScheduledCallback(Handle sch, From d45a2654e5676e189242104ad1f5442279113fd0 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Tue, 5 Aug 2025 12:26:00 -0700 Subject: [PATCH 02/43] First commit --- filament/backend/include/backend/Platform.h | 3 + .../backend/platforms/VulkanPlatform.h | 4 ++ filament/backend/src/vulkan/VulkanDriver.cpp | 67 ++++++++++++++++++- .../src/vulkan/VulkanExternalImageManager.cpp | 21 ++++++ .../src/vulkan/VulkanExternalImageManager.h | 15 +++++ filament/backend/src/vulkan/VulkanTexture.h | 21 ++++++ .../vulkan/platform/VulkanPlatformAndroid.cpp | 5 ++ 7 files changed, 134 insertions(+), 2 deletions(-) diff --git a/filament/backend/include/backend/Platform.h b/filament/backend/include/backend/Platform.h index 20f309069559..844f62e36c53 100644 --- a/filament/backend/include/backend/Platform.h +++ b/filament/backend/include/backend/Platform.h @@ -66,6 +66,9 @@ class UTILS_PUBLIC Platform { ExternalImageHandle& operator=(ExternalImageHandle const& rhs) noexcept; ExternalImageHandle& operator=(ExternalImageHandle&& rhs) noexcept; + bool operator==(const ExternalImageHandle& rhs) const noexcept { + return mTarget == rhs.mTarget; + } explicit operator bool() const noexcept { return mTarget != nullptr; } ExternalImage* UTILS_NULLABLE get() noexcept { return mTarget; } diff --git a/filament/backend/include/backend/platforms/VulkanPlatform.h b/filament/backend/include/backend/platforms/VulkanPlatform.h index b708d8acf766..2cbd34cd8439 100644 --- a/filament/backend/include/backend/platforms/VulkanPlatform.h +++ b/filament/backend/include/backend/platforms/VulkanPlatform.h @@ -385,6 +385,10 @@ class VulkanPlatform : public Platform, utils::PrivateImplementationgetStream(), params); + mAppState.hasBoundExternalImages = true;// still set to true because will be true } else { VulkanSamplerCache::Params cacheParams = { .sampler = params, @@ -956,6 +959,11 @@ void VulkanDriver::destroySwapChain(Handle sch) { } void VulkanDriver::destroyStream(Handle sh) { + if (!sh) { + return; + } + auto stream = resource_ptr::cast(&mResourceManager, sh); + stream.dec(); } void VulkanDriver::destroyTimerQuery(Handle tqh) { @@ -982,18 +990,68 @@ void VulkanDriver::destroyDescriptorSet(Handle dsh) { } Handle VulkanDriver::createStreamNative(void* nativeStream) { - return {}; + return mResourceManager.allocHandle(); } Handle VulkanDriver::createStreamAcquired() { - return {}; + return mResourceManager.allocHandle(); } void VulkanDriver::setAcquiredImage(Handle sh, void* image, const math::mat3f& transform, CallbackHandler* handler, StreamCallback cb, void* userData) { + FVK_SYSTRACE_SCOPE(); + auto stream = resource_ptr::cast(&mResourceManager, sh); + + auto frame = stream->getFrame(image); + if (frame) { + // If this is a new frame for the stream + auto externalImage = mPlatform->createExternalImage(image, false); + auto metadata = mPlatform->extractExternalImageMetadata(externalImage); + auto imgData = mPlatform->createVkImageFromExternal(externalImage); + + assert_invariant(imgData.internal.valid() || imgData.external.valid()); + + VkFormat vkformat = metadata.format; + VkImage vkimage = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + if (imgData.internal.valid()) { + metadata.externalFormat = 0; + vkimage = imgData.internal.image; + memory = imgData.internal.memory; + } else { + vkformat = VK_FORMAT_UNDEFINED; + vkimage = imgData.external.image; + memory = imgData.external.memory; + } + + VkSamplerYcbcrConversion const conversion = + mExternalImageManager.getVkSamplerYcbcrConversion(metadata); + + auto texture = resource_ptr::construct(&mResourceManager, mContext, + mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, vkimage, memory, + vkformat, conversion, metadata.samples, metadata.width, metadata.height, + metadata.layers, metadata.filamentUsage, mStagePool); + auto& commands = mCommands.get(); + // Unlike uploaded textures or swapchains, we need to explicit transition this + // texture into the read layout. + texture->transitionLayout(&commands, texture->getPrimaryViewRange(), + VulkanLayout::FRAG_READ); + + if (imgData.external.valid()) { + stream->pushFrame(image, texture); + mExternalImageManager.addExternallySampledTexture(texture, externalImage); + } + + texture.inc(); + } + + mExternalImageManager.bindStreamFrame(stream, frame); } void VulkanDriver::setStreamDimensions(Handle sh, uint32_t width, uint32_t height) { + auto stream = resource_ptr::cast(&mResourceManager, sh); + stream->width = width; + stream->height = height; } int64_t VulkanDriver::getStreamTimestamp(Handle sh) { @@ -1322,6 +1380,11 @@ TimerQueryResult VulkanDriver::getTimerQueryValue(Handle tqh, uint } void VulkanDriver::setExternalStream(Handle th, Handle sh) { + auto t = resource_ptr::cast(&mResourceManager, th); + assert_invariant(t); + auto s = resource_ptr::cast(&mResourceManager, sh); + assert_invariant(s); + t->setStream(s); } void VulkanDriver::generateMipmaps(Handle th) { diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index 8b1294d56a95..a9a96a472757 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -91,6 +91,7 @@ VulkanExternalImageManager::~VulkanExternalImageManager() = default; void VulkanExternalImageManager::terminate() { mSetBindings.clear(); + mSetSreamBindings.clear(); mImages.clear(); } @@ -262,6 +263,21 @@ void VulkanExternalImageManager::bindExternallySampledTexture( mSetBindings.push_back({ bindingPoint, imageData.image, set, samplerParams }); } +void VulkanExternalImageManager::bindStream( + fvkmemory::resource_ptr set, uint8_t bindingPoint, fvkmemory::resource_ptr stream, + SamplerParams samplerParams) { + mSetSreamBindings.push_back({ bindingPoint, stream, set, samplerParams }); +} + +void VulkanExternalImageManager::bindStreamFrame(fvkmemory::resource_ptr stream, + fvkmemory::resource_ptr frame){ + auto it = std::find_if(mSetSreamBindings.begin(), mSetSreamBindings.end(), + [&](const auto& slot) { return slot.stream == stream; }); + // We should have had this image already bound as a pseudo binding + assert(it != mSetSreamBindings.end()); + bindExternallySampledTexture(it->set, it->binding, frame, it->samplerParams); +} + void VulkanExternalImageManager::addExternallySampledTexture( fvkmemory::resource_ptr image, Platform::ExternalImageHandleRef platformHandleRef) { @@ -277,6 +293,11 @@ void VulkanExternalImageManager::removeExternallySampledTexture( }); } +bool VulkanExternalImageManager::isStreamedTexture( + fvkmemory::resource_ptr image) const { + return bool(image->getStream()); +} + bool VulkanExternalImageManager::isExternallySampledTexture( fvkmemory::resource_ptr image) const { return std::find_if(mImages.begin(), mImages.end(), [&](auto const& imageData) { diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.h b/filament/backend/src/vulkan/VulkanExternalImageManager.h index 543a310d046d..eb663b37fcc2 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.h +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.h @@ -65,6 +65,8 @@ class VulkanExternalImageManager { void bindExternallySampledTexture(fvkmemory::resource_ptr set, uint8_t bindingPoint, fvkmemory::resource_ptr image, SamplerParams samplerParams); + void bindStream(fvkmemory::resource_ptr set, uint8_t bindingPoint, + fvkmemory::resource_ptr stream, SamplerParams samplerParams); void clearTextureBinding(fvkmemory::resource_ptr set, uint8_t bindingPoint); @@ -75,6 +77,11 @@ class VulkanExternalImageManager { void removeExternallySampledTexture(fvkmemory::resource_ptr image); bool isExternallySampledTexture(fvkmemory::resource_ptr image) const; + bool isStreamedTexture(fvkmemory::resource_ptr image) const; + + // For a stream backed VulkanTexture, we are receiving new frames periodically, add them to the tracking system + void bindStreamFrame(fvkmemory::resource_ptr stream, + fvkmemory::resource_ptr frame); VkSamplerYcbcrConversion getVkSamplerYcbcrConversion( VulkanPlatform::ExternalImageMetadata const& metadata); @@ -110,9 +117,17 @@ class VulkanExternalImageManager { SamplerParams samplerParams; bool bound = false; }; + struct SetStreamBindingInfo { + uint8_t binding = 0; + fvkmemory::resource_ptr stream; + fvkmemory::resource_ptr set; + SamplerParams samplerParams; + bool bound = false; + }; // Use vectors instead of hash maps because we only expect small number of entries. std::vector mSetBindings; + std::vector mSetSreamBindings; std::vector mImages; }; diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index 02c88861d0ce..0e3cc28ba7fb 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -36,6 +36,19 @@ namespace filament::backend { struct VulkanTexture; +struct VulkanStream : public HwStream, fvkmemory::ThreadSafeResource { + fvkmemory::resource_ptr getFrame(void* ahb) { + fvkmemory::resource_ptr frame; + const auto& it = mFrames.find(ahb); + if (it != mFrames.end()) frame = it->second; + return frame; + } + void pushFrame(void* ahb, fvkmemory::resource_ptr tex) { mFrames[ahb] = tex; } + +private: + std::map> mFrames; +}; + struct VulkanTextureState : public fvkmemory::Resource { VulkanTextureState(VulkanStagePool& stagePool, VulkanCommands* commands, VmaAllocator allocator, VkDevice device, VkImage image, VkDeviceMemory deviceMemory, VkFormat format, @@ -74,6 +87,8 @@ struct VulkanTextureState : public fvkmemory::Resource { // The texture with the sidecar owns the sidecar. fvkmemory::resource_ptr mSidecarMSAA; + // The stream this texture is associated with (I think cleaner than HwTexture::hwStream). + fvkmemory::resource_ptr mStream; VkImage const mTextureImage; VkDeviceMemory const mTextureImageMemory; @@ -105,6 +120,7 @@ struct VulkanTextureState : public fvkmemory::Resource { using ImageViewHash = utils::hash::MurmurHashFn; std::unordered_map mCachedImageViews; + friend struct VulkanTexture; }; @@ -182,6 +198,11 @@ struct VulkanTexture : public HwTexture, fvkmemory::Resource { return mState->mSidecarMSAA; } + void setStream(fvkmemory::resource_ptr stream) { mState->mStream = stream; + } + + fvkmemory::resource_ptr getStream() const { return mState->mStream; } + bool isTransientAttachment() const { return mState->mUsage & VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT; } diff --git a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp index 899891d090fc..378a779fbc97 100644 --- a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp +++ b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp @@ -165,6 +165,11 @@ VulkanPlatformAndroid::ExternalImageVulkanAndroid::~ExternalImageVulkanAndroid() } } +Platform::ExternalImageHandle VulkanPlatformAndroid::createExternalImage(void* buffer, + bool sRGB) noexcept { + return createExternalImage(const_cast(buffer), sRGB); +} + Platform::ExternalImageHandle VulkanPlatformAndroid::createExternalImage( AHardwareBuffer const* buffer, bool sRGB) noexcept { if (__builtin_available(android 26, *)) { From c00afac722db6c1abb4675e5f9b9f870180af171 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 6 Aug 2025 09:03:05 -0700 Subject: [PATCH 03/43] This is not needed --- .../backend/src/vulkan/platform/VulkanPlatformAndroid.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp index 378a779fbc97..899891d090fc 100644 --- a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp +++ b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp @@ -165,11 +165,6 @@ VulkanPlatformAndroid::ExternalImageVulkanAndroid::~ExternalImageVulkanAndroid() } } -Platform::ExternalImageHandle VulkanPlatformAndroid::createExternalImage(void* buffer, - bool sRGB) noexcept { - return createExternalImage(const_cast(buffer), sRGB); -} - Platform::ExternalImageHandle VulkanPlatformAndroid::createExternalImage( AHardwareBuffer const* buffer, bool sRGB) noexcept { if (__builtin_available(android 26, *)) { From 84fc69ed148fefadb085cd782b589244976c8c62 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 6 Aug 2025 09:31:09 -0700 Subject: [PATCH 04/43] Bypass the AHB* issue. --- filament/backend/include/backend/platforms/VulkanPlatform.h | 3 +++ .../include/backend/platforms/VulkanPlatformAndroid.h | 2 ++ filament/backend/src/vulkan/VulkanDriver.cpp | 2 +- .../backend/src/vulkan/platform/VulkanPlatformAndroid.cpp | 5 +++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/filament/backend/include/backend/platforms/VulkanPlatform.h b/filament/backend/include/backend/platforms/VulkanPlatform.h index 2cbd34cd8439..a83e9989a1d4 100644 --- a/filament/backend/include/backend/platforms/VulkanPlatform.h +++ b/filament/backend/include/backend/platforms/VulkanPlatform.h @@ -410,6 +410,9 @@ class VulkanPlatform : public Platform, utils::PrivateImplementation sh, void* image, const math auto frame = stream->getFrame(image); if (frame) { // If this is a new frame for the stream - auto externalImage = mPlatform->createExternalImage(image, false); + auto externalImage = mPlatform->createExternalImageFromRaw(image, false); auto metadata = mPlatform->extractExternalImageMetadata(externalImage); auto imgData = mPlatform->createVkImageFromExternal(externalImage); diff --git a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp index 899891d090fc..ff7982188b0b 100644 --- a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp +++ b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp @@ -165,6 +165,11 @@ VulkanPlatformAndroid::ExternalImageVulkanAndroid::~ExternalImageVulkanAndroid() } } +Platform::ExternalImageHandle VulkanPlatformAndroid::createExternalImageFromRaw(void* image, + bool sRGB) noexcept { + return createExternalImage(reinterpret_cast(image), sRGB); +} + Platform::ExternalImageHandle VulkanPlatformAndroid::createExternalImage( AHardwareBuffer const* buffer, bool sRGB) noexcept { if (__builtin_available(android 26, *)) { From 1b4f88b1de94a1d7be8977065544bc922f270de6 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 6 Aug 2025 09:35:12 -0700 Subject: [PATCH 05/43] Typo: Forgot to remove the function. --- filament/backend/include/backend/platforms/VulkanPlatform.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/filament/backend/include/backend/platforms/VulkanPlatform.h b/filament/backend/include/backend/platforms/VulkanPlatform.h index a83e9989a1d4..3b614bd669ce 100644 --- a/filament/backend/include/backend/platforms/VulkanPlatform.h +++ b/filament/backend/include/backend/platforms/VulkanPlatform.h @@ -385,10 +385,6 @@ class VulkanPlatform : public Platform, utils::PrivateImplementation Date: Wed, 6 Aug 2025 09:37:42 -0700 Subject: [PATCH 06/43] Typo: No const specifier. --- filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp index ff7982188b0b..c1bccc48ba69 100644 --- a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp +++ b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp @@ -166,7 +166,7 @@ VulkanPlatformAndroid::ExternalImageVulkanAndroid::~ExternalImageVulkanAndroid() } Platform::ExternalImageHandle VulkanPlatformAndroid::createExternalImageFromRaw(void* image, - bool sRGB) noexcept { + bool sRGB) const noexcept { return createExternalImage(reinterpret_cast(image), sRGB); } From 397282a69248f169a48798917396700f5b395e55 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 6 Aug 2025 09:41:14 -0700 Subject: [PATCH 07/43] Removing the const. --- filament/backend/include/backend/platforms/VulkanPlatform.h | 2 +- .../backend/include/backend/platforms/VulkanPlatformAndroid.h | 2 +- filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/filament/backend/include/backend/platforms/VulkanPlatform.h b/filament/backend/include/backend/platforms/VulkanPlatform.h index 3b614bd669ce..02b97eefb7f1 100644 --- a/filament/backend/include/backend/platforms/VulkanPlatform.h +++ b/filament/backend/include/backend/platforms/VulkanPlatform.h @@ -406,7 +406,7 @@ class VulkanPlatform : public Platform, utils::PrivateImplementation(image), sRGB); } From 75fdacb096ebb5a8a31113b98ac9b06642bfa136 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 6 Aug 2025 09:44:25 -0700 Subject: [PATCH 08/43] no exception --- filament/backend/include/backend/platforms/VulkanPlatform.h | 2 +- .../backend/include/backend/platforms/VulkanPlatformAndroid.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/filament/backend/include/backend/platforms/VulkanPlatform.h b/filament/backend/include/backend/platforms/VulkanPlatform.h index 02b97eefb7f1..24616581097c 100644 --- a/filament/backend/include/backend/platforms/VulkanPlatform.h +++ b/filament/backend/include/backend/platforms/VulkanPlatform.h @@ -406,7 +406,7 @@ class VulkanPlatform : public Platform, utils::PrivateImplementation Date: Mon, 11 Aug 2025 09:48:03 -0700 Subject: [PATCH 09/43] Feedback from discussions with Powei. --- filament/backend/src/vulkan/VulkanDriver.cpp | 143 +++++++++++++------ filament/backend/src/vulkan/VulkanDriver.h | 4 + filament/backend/src/vulkan/VulkanTexture.h | 34 ++++- 3 files changed, 129 insertions(+), 52 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 1aeed9cdabd6..da113fa8ebfa 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1001,51 +1001,53 @@ void VulkanDriver::setAcquiredImage(Handle sh, void* image, const math CallbackHandler* handler, StreamCallback cb, void* userData) { FVK_SYSTRACE_SCOPE(); auto stream = resource_ptr::cast(&mResourceManager, sh); - - auto frame = stream->getFrame(image); - if (frame) { - // If this is a new frame for the stream - auto externalImage = mPlatform->createExternalImageFromRaw(image, false); - auto metadata = mPlatform->extractExternalImageMetadata(externalImage); - auto imgData = mPlatform->createVkImageFromExternal(externalImage); - - assert_invariant(imgData.internal.valid() || imgData.external.valid()); - - VkFormat vkformat = metadata.format; - VkImage vkimage = VK_NULL_HANDLE; - VkDeviceMemory memory = VK_NULL_HANDLE; - if (imgData.internal.valid()) { - metadata.externalFormat = 0; - vkimage = imgData.internal.image; - memory = imgData.internal.memory; - } else { - vkformat = VK_FORMAT_UNDEFINED; - vkimage = imgData.external.image; - memory = imgData.external.memory; - } - - VkSamplerYcbcrConversion const conversion = - mExternalImageManager.getVkSamplerYcbcrConversion(metadata); - - auto texture = resource_ptr::construct(&mResourceManager, mContext, - mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, vkimage, memory, - vkformat, conversion, metadata.samples, metadata.width, metadata.height, - metadata.layers, metadata.filamentUsage, mStagePool); - auto& commands = mCommands.get(); - // Unlike uploaded textures or swapchains, we need to explicit transition this - // texture into the read layout. - texture->transitionLayout(&commands, texture->getPrimaryViewRange(), - VulkanLayout::FRAG_READ); - - if (imgData.external.valid()) { - stream->pushFrame(image, texture); - mExternalImageManager.addExternallySampledTexture(texture, externalImage); - } - - texture.inc(); - } - - mExternalImageManager.bindStreamFrame(stream, frame); + stream->acquire({ image, cb, userData, handler }); + mStreamsWithPendingAcquiredImage.push_back(stream); + + //auto frame = stream->getFrame(image); + //if (frame) { + // // If this is a new frame for the stream + // auto externalImage = mPlatform->createExternalImageFromRaw(image, false); + // auto metadata = mPlatform->extractExternalImageMetadata(externalImage); + // auto imgData = mPlatform->createVkImageFromExternal(externalImage); + + // assert_invariant(imgData.internal.valid() || imgData.external.valid()); + + // VkFormat vkformat = metadata.format; + // VkImage vkimage = VK_NULL_HANDLE; + // VkDeviceMemory memory = VK_NULL_HANDLE; + // if (imgData.internal.valid()) { + // metadata.externalFormat = 0; + // vkimage = imgData.internal.image; + // memory = imgData.internal.memory; + // } else { + // vkformat = VK_FORMAT_UNDEFINED; + // vkimage = imgData.external.image; + // memory = imgData.external.memory; + // } + + // VkSamplerYcbcrConversion const conversion = + // mExternalImageManager.getVkSamplerYcbcrConversion(metadata); + + // auto texture = resource_ptr::construct(&mResourceManager, mContext, + // mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, vkimage, memory, + // vkformat, conversion, metadata.samples, metadata.width, metadata.height, + // metadata.layers, metadata.filamentUsage, mStagePool); + // auto& commands = mCommands.get(); + // // Unlike uploaded textures or swapchains, we need to explicit transition this + // // texture into the read layout. + // texture->transitionLayout(&commands, texture->getPrimaryViewRange(), + // VulkanLayout::FRAG_READ); + + // if (imgData.external.valid()) { + // stream->pushFrame(image, texture); + // mExternalImageManager.addExternallySampledTexture(texture, externalImage); + // } + + // texture.inc(); + //} + + //mExternalImageManager.bindStreamFrame(stream, frame); } void VulkanDriver::setStreamDimensions(Handle sh, uint32_t width, uint32_t height) { @@ -1059,6 +1061,57 @@ int64_t VulkanDriver::getStreamTimestamp(Handle sh) { } void VulkanDriver::updateStreams(CommandStream* driver) { + if (UTILS_UNLIKELY(!mStreamsWithPendingAcquiredImage.empty())) { + for (auto& stream: mStreamsWithPendingAcquiredImage) { + if (stream->previousNeedsRelease()) { + scheduleRelease(stream->takePrevious()); + } + + // If this is a new frame for the stream + auto externalImage = + mPlatform->createExternalImageFromRaw(stream->getAcquired().image, false); + auto metadata = mPlatform->extractExternalImageMetadata(externalImage); + auto imgData = mPlatform->createVkImageFromExternal(externalImage); + + assert_invariant(imgData.internal.valid() || imgData.external.valid()); + + VkFormat vkformat = metadata.format; + VkImage vkimage = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + if (imgData.internal.valid()) { + metadata.externalFormat = 0; + vkimage = imgData.internal.image; + memory = imgData.internal.memory; + } else { + vkformat = VK_FORMAT_UNDEFINED; + vkimage = imgData.external.image; + memory = imgData.external.memory; + } + + VkSamplerYcbcrConversion const conversion = + mExternalImageManager.getVkSamplerYcbcrConversion(metadata); + + auto texture = resource_ptr::construct(&mResourceManager, mContext, + mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, vkimage, + memory, vkformat, conversion, metadata.samples, metadata.width, metadata.height, + metadata.layers, metadata.filamentUsage, mStagePool); + auto& commands = mCommands.get(); + // Unlike uploaded textures or swapchains, we need to explicit transition this + // texture into the read layout. + texture->transitionLayout(&commands, texture->getPrimaryViewRange(), + VulkanLayout::FRAG_READ); + + if (imgData.external.valid()) { + // These two now go hand in hand, all new stream images are unique VulkanTextures + // and all require a specific binding to the descriptor set. + mExternalImageManager.addExternallySampledTexture(texture, externalImage); + mExternalImageManager.bindStreamFrame(stream, texture); + } + + texture.inc(); + } + mStreamsWithPendingAcquiredImage.clear(); + } } void VulkanDriver::destroyFence(Handle fh) { diff --git a/filament/backend/src/vulkan/VulkanDriver.h b/filament/backend/src/vulkan/VulkanDriver.h index 2946bf73e779..b8369fd0fbbb 100644 --- a/filament/backend/src/vulkan/VulkanDriver.h +++ b/filament/backend/src/vulkan/VulkanDriver.h @@ -188,6 +188,10 @@ class VulkanDriver final : public DriverBase { bool const mIsSRGBSwapChainSupported; backend::StereoscopicType const mStereoscopicType; + + // setAcquiredImage is a DECL_DRIVER_API_SYNCHRONOUS_N which means we don't necessarily have the + // data to process it at call time. So we store it and process it during updateStreams. + std::vector> mStreamsWithPendingAcquiredImage; }; } // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index 0e3cc28ba7fb..c1467d40cecc 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -37,16 +37,36 @@ namespace filament::backend { struct VulkanTexture; struct VulkanStream : public HwStream, fvkmemory::ThreadSafeResource { - fvkmemory::resource_ptr getFrame(void* ahb) { - fvkmemory::resource_ptr frame; - const auto& it = mFrames.find(ahb); - if (it != mFrames.end()) frame = it->second; - return frame; +// fvkmemory::resource_ptr getFrame(void* ahb) { +// fvkmemory::resource_ptr frame; +// const auto& it = mFrames.find(ahb); +// if (it != mFrames.end()) frame = it->second; +// return frame; +// } +// void pushFrame(void* ahb, fvkmemory::resource_ptr tex) { mFrames[ahb] = tex; } +// +//private: +// std::map> mFrames; +public: + void acquire(const AcquiredImage& image) { + mPrevious = mAcquired; + mAcquired = image; } - void pushFrame(void* ahb, fvkmemory::resource_ptr tex) { mFrames[ahb] = tex; } + bool previousNeedsRelease() const { + return (mPrevious.image != nullptr); + } + // this function will null the previouce once the caller takes it. + // It ensures we don't schedule for release twice. + AcquiredImage takePrevious() { + AcquiredImage previous = mPrevious; + mPrevious = { nullptr, nullptr, nullptr, nullptr }; + return previous; + } + const AcquiredImage& getAcquired()const { return mAcquired; } private: - std::map> mFrames; + AcquiredImage mAcquired; + AcquiredImage mPrevious; }; struct VulkanTextureState : public fvkmemory::Resource { From 3c93f000e29dc6835eb7f372b2ab7e95f903ba59 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Tue, 12 Aug 2025 08:21:34 -0700 Subject: [PATCH 10/43] Making the changes discussed with the team. --- filament/backend/src/vulkan/VulkanDriver.cpp | 86 +++++++++++--------- filament/backend/src/vulkan/VulkanTexture.h | 27 +++--- 2 files changed, 58 insertions(+), 55 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index da113fa8ebfa..e8793e5a29e7 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1067,48 +1067,56 @@ void VulkanDriver::updateStreams(CommandStream* driver) { scheduleRelease(stream->takePrevious()); } - // If this is a new frame for the stream - auto externalImage = - mPlatform->createExternalImageFromRaw(stream->getAcquired().image, false); - auto metadata = mPlatform->extractExternalImageMetadata(externalImage); - auto imgData = mPlatform->createVkImageFromExternal(externalImage); - - assert_invariant(imgData.internal.valid() || imgData.external.valid()); - - VkFormat vkformat = metadata.format; - VkImage vkimage = VK_NULL_HANDLE; - VkDeviceMemory memory = VK_NULL_HANDLE; - if (imgData.internal.valid()) { - metadata.externalFormat = 0; - vkimage = imgData.internal.image; - memory = imgData.internal.memory; - } else { - vkformat = VK_FORMAT_UNDEFINED; - vkimage = imgData.external.image; - memory = imgData.external.memory; - } + auto texture = stream->getTexture(stream->getAcquired().image); + if (!texture) { + auto externalImage = + mPlatform->createExternalImageFromRaw(stream->getAcquired().image, false); + auto metadata = mPlatform->extractExternalImageMetadata(externalImage); + auto imgData = mPlatform->createVkImageFromExternal(externalImage); + + assert_invariant(imgData.internal.valid() || imgData.external.valid()); + + VkFormat vkformat = metadata.format; + VkImage vkimage = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + if (imgData.internal.valid()) { + metadata.externalFormat = 0; + vkimage = imgData.internal.image; + memory = imgData.internal.memory; + } else { + vkformat = VK_FORMAT_UNDEFINED; + vkimage = imgData.external.image; + memory = imgData.external.memory; + } + + VkSamplerYcbcrConversion const conversion = + mExternalImageManager.getVkSamplerYcbcrConversion(metadata); + + auto newTexture = resource_ptr::construct(&mResourceManager, mContext, + mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, vkimage, + memory, vkformat, conversion, metadata.samples, metadata.width, + metadata.height, metadata.layers, metadata.filamentUsage, mStagePool); + + // We shouldn't need to do this when we allocate the texture + if (true) { + auto& commands = mCommands.get(); + // Unlike uploaded textures or swapchains, we need to explicit transition this + // texture into the read layout. + newTexture->transitionLayout(&commands, newTexture->getPrimaryViewRange(), + VulkanLayout::FRAG_READ); + } + + if (imgData.external.valid()) { + mExternalImageManager.addExternallySampledTexture(newTexture, externalImage); + // Cache the AHB backed image. + stream->pushImage(stream->getAcquired().image, newTexture); + } - VkSamplerYcbcrConversion const conversion = - mExternalImageManager.getVkSamplerYcbcrConversion(metadata); - - auto texture = resource_ptr::construct(&mResourceManager, mContext, - mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, vkimage, - memory, vkformat, conversion, metadata.samples, metadata.width, metadata.height, - metadata.layers, metadata.filamentUsage, mStagePool); - auto& commands = mCommands.get(); - // Unlike uploaded textures or swapchains, we need to explicit transition this - // texture into the read layout. - texture->transitionLayout(&commands, texture->getPrimaryViewRange(), - VulkanLayout::FRAG_READ); - - if (imgData.external.valid()) { - // These two now go hand in hand, all new stream images are unique VulkanTextures - // and all require a specific binding to the descriptor set. - mExternalImageManager.addExternallySampledTexture(texture, externalImage); - mExternalImageManager.bindStreamFrame(stream, texture); + newTexture.inc(); + texture = newTexture; } - texture.inc(); + mExternalImageManager.bindStreamFrame(stream, texture); } mStreamsWithPendingAcquiredImage.clear(); } diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index c1467d40cecc..05065b6a9144 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -37,36 +37,31 @@ namespace filament::backend { struct VulkanTexture; struct VulkanStream : public HwStream, fvkmemory::ThreadSafeResource { -// fvkmemory::resource_ptr getFrame(void* ahb) { -// fvkmemory::resource_ptr frame; -// const auto& it = mFrames.find(ahb); -// if (it != mFrames.end()) frame = it->second; -// return frame; -// } -// void pushFrame(void* ahb, fvkmemory::resource_ptr tex) { mFrames[ahb] = tex; } -// -//private: -// std::map> mFrames; -public: + fvkmemory::resource_ptr getTexture(void* ahb) { + fvkmemory::resource_ptr frame; + const auto& it = mTextures.find(ahb); + if (it != mTextures.end()) frame = it->second; + return frame; + } + void pushImage(void* ahb, fvkmemory::resource_ptr tex) { mTextures[ahb] = tex; } void acquire(const AcquiredImage& image) { mPrevious = mAcquired; mAcquired = image; } - bool previousNeedsRelease() const { - return (mPrevious.image != nullptr); - } - // this function will null the previouce once the caller takes it. + bool previousNeedsRelease() const { return (mPrevious.image != nullptr); } + // this function will null the previous once the caller takes it. // It ensures we don't schedule for release twice. AcquiredImage takePrevious() { AcquiredImage previous = mPrevious; mPrevious = { nullptr, nullptr, nullptr, nullptr }; return previous; } - const AcquiredImage& getAcquired()const { return mAcquired; } + const AcquiredImage& getAcquired() const { return mAcquired; } private: AcquiredImage mAcquired; AcquiredImage mPrevious; + std::map> mTextures; }; struct VulkanTextureState : public fvkmemory::Resource { From 8b6fa40fd6b34c24709c202a5d166a05c0edc2e5 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Tue, 12 Aug 2025 08:32:47 -0700 Subject: [PATCH 11/43] Removing useless comments. --- filament/backend/src/vulkan/VulkanDriver.cpp | 45 -------------------- 1 file changed, 45 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 5e43183b1800..cca7bc8d2f72 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1012,51 +1012,6 @@ void VulkanDriver::setAcquiredImage(Handle sh, void* image, const math auto stream = resource_ptr::cast(&mResourceManager, sh); stream->acquire({ image, cb, userData, handler }); mStreamsWithPendingAcquiredImage.push_back(stream); - - //auto frame = stream->getFrame(image); - //if (frame) { - // // If this is a new frame for the stream - // auto externalImage = mPlatform->createExternalImageFromRaw(image, false); - // auto metadata = mPlatform->extractExternalImageMetadata(externalImage); - // auto imgData = mPlatform->createVkImageFromExternal(externalImage); - - // assert_invariant(imgData.internal.valid() || imgData.external.valid()); - - // VkFormat vkformat = metadata.format; - // VkImage vkimage = VK_NULL_HANDLE; - // VkDeviceMemory memory = VK_NULL_HANDLE; - // if (imgData.internal.valid()) { - // metadata.externalFormat = 0; - // vkimage = imgData.internal.image; - // memory = imgData.internal.memory; - // } else { - // vkformat = VK_FORMAT_UNDEFINED; - // vkimage = imgData.external.image; - // memory = imgData.external.memory; - // } - - // VkSamplerYcbcrConversion const conversion = - // mExternalImageManager.getVkSamplerYcbcrConversion(metadata); - - // auto texture = resource_ptr::construct(&mResourceManager, mContext, - // mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, vkimage, memory, - // vkformat, conversion, metadata.samples, metadata.width, metadata.height, - // metadata.layers, metadata.filamentUsage, mStagePool); - // auto& commands = mCommands.get(); - // // Unlike uploaded textures or swapchains, we need to explicit transition this - // // texture into the read layout. - // texture->transitionLayout(&commands, texture->getPrimaryViewRange(), - // VulkanLayout::FRAG_READ); - - // if (imgData.external.valid()) { - // stream->pushFrame(image, texture); - // mExternalImageManager.addExternallySampledTexture(texture, externalImage); - // } - - // texture.inc(); - //} - - //mExternalImageManager.bindStreamFrame(stream, frame); } void VulkanDriver::setStreamDimensions(Handle sh, uint32_t width, uint32_t height) { From 76fc21a984372350c1fc8927779a88c65ca83754 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 13 Aug 2025 18:06:27 -0700 Subject: [PATCH 12/43] Adding the required methods for allocation. --- filament/backend/src/vulkan/VulkanDriver.cpp | 10 ++++++++-- filament/backend/src/vulkan/memory/Resource.cpp | 6 ++++++ filament/backend/src/vulkan/memory/Resource.h | 3 ++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index cca7bc8d2f72..b7f17fe6d8ac 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -999,11 +999,17 @@ void VulkanDriver::destroyDescriptorSet(Handle dsh) { } Handle VulkanDriver::createStreamNative(void* nativeStream) { - return mResourceManager.allocHandle(); + auto handle = mResourceManager.allocHandle(); + auto stream = resource_ptr::make(&mResourceManager, handle); + stream.inc(); + return handle; } Handle VulkanDriver::createStreamAcquired() { - return mResourceManager.allocHandle(); + auto handle = mResourceManager.allocHandle(); + auto stream = resource_ptr::make(&mResourceManager, handle); + stream.inc(); + return handle; } void VulkanDriver::setAcquiredImage(Handle sh, void* image, const math::mat3f& transform, diff --git a/filament/backend/src/vulkan/memory/Resource.cpp b/filament/backend/src/vulkan/memory/Resource.cpp index 13669e9a9a77..38c3620df4d6 100644 --- a/filament/backend/src/vulkan/memory/Resource.cpp +++ b/filament/backend/src/vulkan/memory/Resource.cpp @@ -38,6 +38,7 @@ template ResourceType getTypeEnum() noexcept; template ResourceType getTypeEnum() noexcept; template ResourceType getTypeEnum() noexcept; template ResourceType getTypeEnum() noexcept; +template ResourceType getTypeEnum() noexcept; template ResourceType getTypeEnum() noexcept { @@ -92,6 +93,9 @@ ResourceType getTypeEnum() noexcept { if constexpr (std::is_same_v) { return ResourceType::VULKAN_BUFFER; } + if constexpr (std::is_same_v) { + return ResourceType::STREAM; + } return ResourceType::UNDEFINED_TYPE; } @@ -131,6 +135,8 @@ std::string getTypeStr(ResourceType type) { return "Fence"; case ResourceType::VULKAN_BUFFER: return "VulkanBuffer"; + case ResourceType::STREAM: + return "VulkanStream"; case ResourceType::UNDEFINED_TYPE: return ""; } diff --git a/filament/backend/src/vulkan/memory/Resource.h b/filament/backend/src/vulkan/memory/Resource.h index ff79a8f79d2e..fbdb5205ca68 100644 --- a/filament/backend/src/vulkan/memory/Resource.h +++ b/filament/backend/src/vulkan/memory/Resource.h @@ -52,7 +52,8 @@ enum class ResourceType : uint8_t { VULKAN_BUFFER = 14, STAGE_SEGMENT = 15, STAGE_IMAGE = 16, - UNDEFINED_TYPE = 17, // Must be the last enum because we use it for iterating over the enums. + STREAM = 17, + UNDEFINED_TYPE = 18, // Must be the last enum because we use it for iterating over the enums. }; template From 5858acf2d349e5c7ec7790078d3112110b011c67 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 13 Aug 2025 18:10:17 -0700 Subject: [PATCH 13/43] Forgot one more switch case. --- filament/backend/src/vulkan/memory/ResourceManager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/filament/backend/src/vulkan/memory/ResourceManager.cpp b/filament/backend/src/vulkan/memory/ResourceManager.cpp index 227c4d84f9ee..8766677de5b1 100644 --- a/filament/backend/src/vulkan/memory/ResourceManager.cpp +++ b/filament/backend/src/vulkan/memory/ResourceManager.cpp @@ -114,6 +114,9 @@ void ResourceManager::destroyWithType(ResourceType type, HandleId id) { case ResourceType::VULKAN_BUFFER: destruct(Handle(id)); break; + case ResourceType::STREAM: + destruct(Handle(id)); + break; case ResourceType::UNDEFINED_TYPE: break; } From e441cbba3a4a8bb0031cf5986faea7d232e8a1fd Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Thu, 14 Aug 2025 18:25:43 -0700 Subject: [PATCH 14/43] =?UTF-8?q?Moving=20the=20resource=20to=20=C2=96fvkm?= =?UTF-8?q?emory::Resource=20type.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- filament/backend/src/vulkan/VulkanTexture.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index 05065b6a9144..b87dad3122a5 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -36,7 +36,7 @@ namespace filament::backend { struct VulkanTexture; -struct VulkanStream : public HwStream, fvkmemory::ThreadSafeResource { +struct VulkanStream : public HwStream, fvkmemory::Resource { fvkmemory::resource_ptr getTexture(void* ahb) { fvkmemory::resource_ptr frame; const auto& it = mTextures.find(ahb); From 865cb0a58dd7775a9c4acf20fc065cad4f4edc1d Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Mon, 18 Aug 2025 11:24:47 -0700 Subject: [PATCH 15/43] Adding streamed set. --- filament/backend/src/vulkan/VulkanExternalImageManager.cpp | 1 + filament/backend/src/vulkan/VulkanHandles.h | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index a9a96a472757..2c37f58bebd4 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -267,6 +267,7 @@ void VulkanExternalImageManager::bindStream( fvkmemory::resource_ptr set, uint8_t bindingPoint, fvkmemory::resource_ptr stream, SamplerParams samplerParams) { mSetSreamBindings.push_back({ bindingPoint, stream, set, samplerParams }); + set->setHasStreamedTexture(); } void VulkanExternalImageManager::bindStreamFrame(fvkmemory::resource_ptr stream, diff --git a/filament/backend/src/vulkan/VulkanHandles.h b/filament/backend/src/vulkan/VulkanHandles.h index a1d6dc03b829..62ba4575ed8e 100644 --- a/filament/backend/src/vulkan/VulkanHandles.h +++ b/filament/backend/src/vulkan/VulkanHandles.h @@ -185,6 +185,9 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { return mExternalSamplerVkSet; } + void setHasStreamedTexture() { mHasStreamedTexture = true; + } + void setExternalSamplerVkSet(VkDescriptorSet vkset, OnRecycle onRecycle) { mExternalSamplerVkSet = vkset; if (mOnRecycleExternalSamplerFn) { @@ -217,6 +220,7 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { private: VkDescriptorSet const mVkSet; VkDescriptorSet mExternalSamplerVkSet = VK_NULL_HANDLE; + bool mHasStreamedTexture = false; backend::DescriptorSetOffsetArray mOffsets; std::vector> mResources; From c6d01271f947dde1b4a780863ab3a4c92c5b20bf Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Mon, 18 Aug 2025 20:36:11 -0700 Subject: [PATCH 16/43] Allocating every frame for streamed textures. --- filament/backend/src/vulkan/VulkanExternalImageManager.cpp | 2 +- filament/backend/src/vulkan/VulkanHandles.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index 7a5f08ebcfde..058a5f74e1b6 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -179,7 +179,7 @@ void VulkanExternalImageManager::updateSetAndLayout( // Need to copy the set VkDescriptorSet const oldSet = set->getExternalSamplerVkSet(); - if (oldLayout != newLayout || oldSet == VK_NULL_HANDLE) { + if (set->getHasStreamedTexture() || (oldLayout != newLayout || oldSet == VK_NULL_HANDLE)) { // Build a new descriptor set from the new layout VkDescriptorSet const newSet = mDescriptorSetCache->getVkSet(layout->count, newLayout); auto const ubo = layout->bitmask.ubo | layout->bitmask.dynamicUbo; diff --git a/filament/backend/src/vulkan/VulkanHandles.h b/filament/backend/src/vulkan/VulkanHandles.h index 62ba4575ed8e..05684826a494 100644 --- a/filament/backend/src/vulkan/VulkanHandles.h +++ b/filament/backend/src/vulkan/VulkanHandles.h @@ -188,6 +188,8 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { void setHasStreamedTexture() { mHasStreamedTexture = true; } + bool getHasStreamedTexture()const { return mHasStreamedTexture; } + void setExternalSamplerVkSet(VkDescriptorSet vkset, OnRecycle onRecycle) { mExternalSamplerVkSet = vkset; if (mOnRecycleExternalSamplerFn) { From 8017d5ee0faa3031493399cc64566f053c83e57c Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 20 Aug 2025 09:54:00 -0700 Subject: [PATCH 17/43] Changing the binding logic. --- filament/backend/src/vulkan/VulkanDriver.cpp | 3 +++ .../src/vulkan/VulkanExternalImageManager.cpp | 21 ++++++++++++------- .../src/vulkan/VulkanExternalImageManager.h | 3 +-- filament/backend/src/vulkan/VulkanHandles.h | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index b7f17fe6d8ac..7222ac0078ca 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -455,9 +455,11 @@ void VulkanDriver::updateDescriptorSetTexture( if (mExternalImageManager.isExternallySampledTexture(texture)) { mExternalImageManager.bindExternallySampledTexture(set, binding, texture, params); mAppState.hasBoundExternalImages = true; + set->setHasStreamedTexture(false); } else if (mExternalImageManager.isStreamedTexture(texture)) { mExternalImageManager.bindStream(set, binding, texture->getStream(), params); mAppState.hasBoundExternalImages = true;// still set to true because will be true + set->setHasStreamedTexture(true); } else { VulkanSamplerCache::Params cacheParams = { .sampler = params, @@ -465,6 +467,7 @@ void VulkanDriver::updateDescriptorSetTexture( VkSampler const vksampler = mSamplerCache.getSampler(cacheParams); mDescriptorSetCache.updateSampler(set, binding, texture, vksampler); mExternalImageManager.clearTextureBinding(set, binding); + set->setHasStreamedTexture(false); } } diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index 058a5f74e1b6..fa48424b5acf 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -91,7 +91,7 @@ VulkanExternalImageManager::~VulkanExternalImageManager() = default; void VulkanExternalImageManager::terminate() { mSetBindings.clear(); - mSetSreamBindings.clear(); + mSetStreamBindings.clear(); mImages.clear(); } @@ -268,17 +268,22 @@ void VulkanExternalImageManager::bindExternallySampledTexture( void VulkanExternalImageManager::bindStream( fvkmemory::resource_ptr set, uint8_t bindingPoint, fvkmemory::resource_ptr stream, SamplerParams samplerParams) { - mSetSreamBindings.push_back({ bindingPoint, stream, set, samplerParams }); - set->setHasStreamedTexture(); + mSetStreamBindings.push_back({ bindingPoint, stream, set, samplerParams }); } void VulkanExternalImageManager::bindStreamFrame(fvkmemory::resource_ptr stream, fvkmemory::resource_ptr frame){ - auto it = std::find_if(mSetSreamBindings.begin(), mSetSreamBindings.end(), - [&](const auto& slot) { return slot.stream == stream; }); - // We should have had this image already bound as a pseudo binding - assert(it != mSetSreamBindings.end()); - bindExternallySampledTexture(it->set, it->binding, frame, it->samplerParams); + auto it = std::find_if(mSetStreamBindings.begin(), mSetStreamBindings.end(), + [&](auto const& streamData) { return streamData.stream == stream; }); + // We should have had this stream already through mSetStreamBindings + assert(it != mSetStreamBindings.end()); + + // If we do not have this image yet, add it + if (std::find_if(mSetBindings.begin(), mSetBindings.end(), [&](auto const& imageData) { + return imageData.image == frame; + }) == mSetBindings.end()) { + mSetBindings.push_back({ it->binding, frame, it->set, it->samplerParams }); + } } void VulkanExternalImageManager::addExternallySampledTexture( diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.h b/filament/backend/src/vulkan/VulkanExternalImageManager.h index eb663b37fcc2..e9ba32485779 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.h +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.h @@ -122,12 +122,11 @@ class VulkanExternalImageManager { fvkmemory::resource_ptr stream; fvkmemory::resource_ptr set; SamplerParams samplerParams; - bool bound = false; }; // Use vectors instead of hash maps because we only expect small number of entries. std::vector mSetBindings; - std::vector mSetSreamBindings; + std::vector mSetStreamBindings; std::vector mImages; }; diff --git a/filament/backend/src/vulkan/VulkanHandles.h b/filament/backend/src/vulkan/VulkanHandles.h index 05684826a494..f905deb07621 100644 --- a/filament/backend/src/vulkan/VulkanHandles.h +++ b/filament/backend/src/vulkan/VulkanHandles.h @@ -185,7 +185,7 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { return mExternalSamplerVkSet; } - void setHasStreamedTexture() { mHasStreamedTexture = true; + void setHasStreamedTexture(bool does) { mHasStreamedTexture = does; } bool getHasStreamedTexture()const { return mHasStreamedTexture; } From ad57838f18cbb8de544468a30a7d7952afe1a312 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Mon, 25 Aug 2025 09:27:54 -0700 Subject: [PATCH 18/43] Proper checks for the streamed texture. --- .../src/vulkan/VulkanExternalImageManager.cpp | 18 ++++++++++++------ .../src/vulkan/VulkanExternalImageManager.h | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index fa48424b5acf..a95d2d6c3f08 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -113,8 +113,9 @@ fvkutils::DescriptorSetMask VulkanExternalImageManager::prepareBindSets(LayoutAr if (!set || !layout) { continue; } - if (hasExternalSampler(set)) { - updateSetAndLayout(set, layout); + bool streamed = false; + if (hasExternalSampler(set, streamed)) { + updateSetAndLayout(set, layout, streamed); shouldUseExternalSampler.set(i); } } @@ -122,15 +123,20 @@ fvkutils::DescriptorSetMask VulkanExternalImageManager::prepareBindSets(LayoutAr } bool VulkanExternalImageManager::hasExternalSampler( - fvkmemory::resource_ptr set) { + fvkmemory::resource_ptr set, bool& streamed) { auto itr = std::find_if(mSetBindings.begin(), mSetBindings.end(), [&](SetBindingInfo const& info) { return info.set == set; }); - return itr != mSetBindings.end(); + bool doesIt = false; + if (itr != mSetBindings.end()) { + streamed = bool(itr->image->getStream()); + doesIt = true; + } + return doesIt; } void VulkanExternalImageManager::updateSetAndLayout( fvkmemory::resource_ptr set, - fvkmemory::resource_ptr layout) { + fvkmemory::resource_ptr layout, bool streamed) { utils::FixedCapacityVector< std::tuple>> samplerAndBindings; @@ -179,7 +185,7 @@ void VulkanExternalImageManager::updateSetAndLayout( // Need to copy the set VkDescriptorSet const oldSet = set->getExternalSamplerVkSet(); - if (set->getHasStreamedTexture() || (oldLayout != newLayout || oldSet == VK_NULL_HANDLE)) { + if (streamed || (oldLayout != newLayout || oldSet == VK_NULL_HANDLE)) { // Build a new descriptor set from the new layout VkDescriptorSet const newSet = mDescriptorSetCache->getVkSet(layout->count, newLayout); auto const ubo = layout->bitmask.ubo | layout->bitmask.dynamicUbo; diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.h b/filament/backend/src/vulkan/VulkanExternalImageManager.h index e9ba32485779..6c546a076746 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.h +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.h @@ -94,10 +94,10 @@ class VulkanExternalImageManager { }; private: - bool hasExternalSampler(fvkmemory::resource_ptr set); + bool hasExternalSampler(fvkmemory::resource_ptr set, bool& streamed); void updateSetAndLayout(fvkmemory::resource_ptr set, - fvkmemory::resource_ptr layout); + fvkmemory::resource_ptr layout, bool streamed); void updateImage(ImageData* imageData); From 29e3f07a6be0d2d7e71765c9afc6ebea9fc33385 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Mon, 25 Aug 2025 13:28:24 -0700 Subject: [PATCH 19/43] Cleanup. --- filament/backend/src/vulkan/VulkanDriver.cpp | 5 +++-- .../backend/src/vulkan/VulkanExternalImageManager.cpp | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 21f559e78de2..9797028a6905 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -458,7 +458,7 @@ void VulkanDriver::updateDescriptorSetTexture( set->setHasStreamedTexture(false); } else if (mExternalImageManager.isStreamedTexture(texture)) { mExternalImageManager.bindStream(set, binding, texture->getStream(), params); - mAppState.hasBoundExternalImages = true;// still set to true because will be true + mAppState.hasBoundExternalImages = true;// because there will always be external images set->setHasStreamedTexture(true); } else { VulkanSamplerCache::Params cacheParams = { @@ -2034,8 +2034,9 @@ void VulkanDriver::bindDescriptorSet( // been bound and this set has external samplers, we do the doBindindraw block in // draw2() again. Because this set might potentially cause a new pipelineLayout // (therefore pipeline) to be bound. + bool streamed; if (bindInDrawBundle.descriptorSetMask[setIndex] && - mExternalImageManager.hasExternalSampler(set)) { + mExternalImageManager.hasExternalSampler(set, streamed)) { mPipelineState.bindInDraw.first = true; } } diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index a95d2d6c3f08..08a9bf272c89 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -123,12 +123,16 @@ fvkutils::DescriptorSetMask VulkanExternalImageManager::prepareBindSets(LayoutAr } bool VulkanExternalImageManager::hasExternalSampler( - fvkmemory::resource_ptr set, bool& streamed) { + fvkmemory::resource_ptr set, bool& streamed) const { auto itr = std::find_if(mSetBindings.begin(), mSetBindings.end(), [&](SetBindingInfo const& info) { return info.set == set; }); bool doesIt = false; + streamed = false; + if (itr != mSetBindings.end()) { - streamed = bool(itr->image->getStream()); + auto itrStream = std::find_if(mSetStreamBindings.begin(), mSetStreamBindings.end(), + [&](SetStreamBindingInfo const& info) { return info.set == set; }); + streamed = (itrStream != mSetStreamBindings.end()); doesIt = true; } return doesIt; From 406a348a7d6a5ac86c5613c12d2c403717801506 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 27 Aug 2025 19:17:55 -0700 Subject: [PATCH 20/43] Chaning the logic of the state tracking for streamed textures. --- filament/backend/src/vulkan/VulkanDriver.cpp | 49 +++++++++++++++---- filament/backend/src/vulkan/VulkanDriver.h | 16 ++++++ .../src/vulkan/VulkanExternalImageManager.cpp | 49 +++---------------- .../src/vulkan/VulkanExternalImageManager.h | 18 +------ filament/backend/src/vulkan/VulkanHandles.h | 6 --- 5 files changed, 63 insertions(+), 75 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 9797028a6905..8f7cb050ee71 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -378,6 +378,35 @@ void VulkanDriver::tick(int) { mCommands.updateFences(); } +void VulkanDriver::bindStreamedTexture(fvkmemory::resource_ptr set, + uint8_t bindingPoint, fvkmemory::resource_ptr image, + SamplerParams samplerParams) { + mStreamedTexturesBindings.push_back({ bindingPoint, image, set, samplerParams }); +} + +void VulkanDriver::unbindStreamedTexture(fvkmemory::resource_ptr set, + uint8_t bindingPoint) { + std::erase_if(mStreamedTexturesBindings, [&](streamedTextureBinding& binding) { + return ((binding.set == set) && (binding.binding == bindingPoint)); + }); +} + +void VulkanDriver::onStreamAcquireImage(fvkmemory::resource_ptr image, + fvkmemory::resource_ptr stream, bool newImage) { + for (streamedTextureBinding const& data: mStreamedTexturesBindings) { + if (data.image->getStream() == stream) { + if (newImage) { + mExternalImageManager.bindExternallySampledTexture(data.set, data.binding, image, + data.samplerParams); + } else { + // just update the sampler set + mDescriptorSetCache.updateSamplerForExternalSamplerSet(data.set, data.binding, + image); + } + } + } +} + // Garbage collection should not occur too frequently, only about once per frame. Internally, the // eviction time of various resources is often measured in terms of an approximate frame number // rather than the wall clock, because we must wait 3 frames after a DriverAPI-level resource has @@ -455,19 +484,18 @@ void VulkanDriver::updateDescriptorSetTexture( if (UTILS_UNLIKELY(mExternalImageManager.isExternallySampledTexture(texture))) { mExternalImageManager.bindExternallySampledTexture(set, binding, texture, params); mAppState.hasBoundExternalImages = true; - set->setHasStreamedTexture(false); - } else if (mExternalImageManager.isStreamedTexture(texture)) { - mExternalImageManager.bindStream(set, binding, texture->getStream(), params); - mAppState.hasBoundExternalImages = true;// because there will always be external images - set->setHasStreamedTexture(true); - } else { + } else if (bool(texture->getStream())) { + bindStreamedTexture(set, binding, texture, params); + mAppState.hasBoundExternalImages = true; + } + else { VulkanSamplerCache::Params cacheParams = { .sampler = params, }; VkSampler const vksampler = mSamplerCache.getSampler(cacheParams); mDescriptorSetCache.updateSampler(set, binding, texture, vksampler); mExternalImageManager.clearTextureBinding(set, binding); - set->setHasStreamedTexture(false); + unbindStreamedTexture(set, binding); } } @@ -1041,6 +1069,7 @@ void VulkanDriver::updateStreams(CommandStream* driver) { } auto texture = stream->getTexture(stream->getAcquired().image); + bool newImage = false; if (!texture) { auto externalImage = mPlatform->createExternalImageFromRaw(stream->getAcquired().image, false); @@ -1083,13 +1112,14 @@ void VulkanDriver::updateStreams(CommandStream* driver) { mExternalImageManager.addExternallySampledTexture(newTexture, externalImage); // Cache the AHB backed image. stream->pushImage(stream->getAcquired().image, newTexture); + newImage = true; } newTexture.inc(); texture = newTexture; } - mExternalImageManager.bindStreamFrame(stream, texture); + onStreamAcquireImage(texture, stream, newImage); } mStreamsWithPendingAcquiredImage.clear(); } @@ -2034,9 +2064,8 @@ void VulkanDriver::bindDescriptorSet( // been bound and this set has external samplers, we do the doBindindraw block in // draw2() again. Because this set might potentially cause a new pipelineLayout // (therefore pipeline) to be bound. - bool streamed; if (bindInDrawBundle.descriptorSetMask[setIndex] && - mExternalImageManager.hasExternalSampler(set, streamed)) { + mExternalImageManager.hasExternalSampler(set)) { mPipelineState.bindInDraw.first = true; } } diff --git a/filament/backend/src/vulkan/VulkanDriver.h b/filament/backend/src/vulkan/VulkanDriver.h index b8369fd0fbbb..5b459ead992f 100644 --- a/filament/backend/src/vulkan/VulkanDriver.h +++ b/filament/backend/src/vulkan/VulkanDriver.h @@ -121,6 +121,13 @@ class VulkanDriver final : public DriverBase { VulkanDriver& operator=(VulkanDriver const&) = delete; private: + void bindStreamedTexture(fvkmemory::resource_ptr set, uint8_t bindingPoint, + fvkmemory::resource_ptr image, SamplerParams samplerParams); + void unbindStreamedTexture(fvkmemory::resource_ptr set, + uint8_t bindingPoint); + void onStreamAcquireImage(fvkmemory::resource_ptr image, + fvkmemory::resource_ptr stream, bool newImage); + void collectGarbage(); void bindPipelineImpl(PipelineState const& pipelineState, VkPipelineLayout pipelineLayout, fvkutils::DescriptorSetMask descriptorSetMask); @@ -192,6 +199,15 @@ class VulkanDriver final : public DriverBase { // setAcquiredImage is a DECL_DRIVER_API_SYNCHRONOUS_N which means we don't necessarily have the // data to process it at call time. So we store it and process it during updateStreams. std::vector> mStreamsWithPendingAcquiredImage; + + struct streamedTextureBinding { + uint8_t binding = 0; + fvkmemory::resource_ptr image; + fvkmemory::resource_ptr set; + SamplerParams samplerParams; + }; + // keep track of all the stream bindings + std::vector mStreamedTexturesBindings; }; } // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index 08a9bf272c89..886de2a0f22c 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -91,7 +91,6 @@ VulkanExternalImageManager::~VulkanExternalImageManager() = default; void VulkanExternalImageManager::terminate() { mSetBindings.clear(); - mSetStreamBindings.clear(); mImages.clear(); } @@ -113,9 +112,8 @@ fvkutils::DescriptorSetMask VulkanExternalImageManager::prepareBindSets(LayoutAr if (!set || !layout) { continue; } - bool streamed = false; - if (hasExternalSampler(set, streamed)) { - updateSetAndLayout(set, layout, streamed); + if (hasExternalSampler(set)) { + updateSetAndLayout(set, layout); shouldUseExternalSampler.set(i); } } @@ -123,24 +121,15 @@ fvkutils::DescriptorSetMask VulkanExternalImageManager::prepareBindSets(LayoutAr } bool VulkanExternalImageManager::hasExternalSampler( - fvkmemory::resource_ptr set, bool& streamed) const { + fvkmemory::resource_ptr set)const { auto itr = std::find_if(mSetBindings.begin(), mSetBindings.end(), [&](SetBindingInfo const& info) { return info.set == set; }); - bool doesIt = false; - streamed = false; - - if (itr != mSetBindings.end()) { - auto itrStream = std::find_if(mSetStreamBindings.begin(), mSetStreamBindings.end(), - [&](SetStreamBindingInfo const& info) { return info.set == set; }); - streamed = (itrStream != mSetStreamBindings.end()); - doesIt = true; - } - return doesIt; + return itr != mSetBindings.end(); } void VulkanExternalImageManager::updateSetAndLayout( fvkmemory::resource_ptr set, - fvkmemory::resource_ptr layout, bool streamed) { + fvkmemory::resource_ptr layout) { utils::FixedCapacityVector< std::tuple>> samplerAndBindings; @@ -189,7 +178,7 @@ void VulkanExternalImageManager::updateSetAndLayout( // Need to copy the set VkDescriptorSet const oldSet = set->getExternalSamplerVkSet(); - if (streamed || (oldLayout != newLayout || oldSet == VK_NULL_HANDLE)) { + if (oldLayout != newLayout || oldSet == VK_NULL_HANDLE) { // Build a new descriptor set from the new layout VkDescriptorSet const newSet = mDescriptorSetCache->getVkSet(layout->count, newLayout); auto const ubo = layout->bitmask.ubo | layout->bitmask.dynamicUbo; @@ -275,27 +264,6 @@ void VulkanExternalImageManager::bindExternallySampledTexture( mSetBindings.push_back({ bindingPoint, imageData.image, set, samplerParams }); } -void VulkanExternalImageManager::bindStream( - fvkmemory::resource_ptr set, uint8_t bindingPoint, fvkmemory::resource_ptr stream, - SamplerParams samplerParams) { - mSetStreamBindings.push_back({ bindingPoint, stream, set, samplerParams }); -} - -void VulkanExternalImageManager::bindStreamFrame(fvkmemory::resource_ptr stream, - fvkmemory::resource_ptr frame){ - auto it = std::find_if(mSetStreamBindings.begin(), mSetStreamBindings.end(), - [&](auto const& streamData) { return streamData.stream == stream; }); - // We should have had this stream already through mSetStreamBindings - assert(it != mSetStreamBindings.end()); - - // If we do not have this image yet, add it - if (std::find_if(mSetBindings.begin(), mSetBindings.end(), [&](auto const& imageData) { - return imageData.image == frame; - }) == mSetBindings.end()) { - mSetBindings.push_back({ it->binding, frame, it->set, it->samplerParams }); - } -} - void VulkanExternalImageManager::addExternallySampledTexture( fvkmemory::resource_ptr image, Platform::ExternalImageHandleRef platformHandleRef) { @@ -311,11 +279,6 @@ void VulkanExternalImageManager::removeExternallySampledTexture( }); } -bool VulkanExternalImageManager::isStreamedTexture( - fvkmemory::resource_ptr image) const { - return bool(image->getStream()); -} - bool VulkanExternalImageManager::isExternallySampledTexture( fvkmemory::resource_ptr image) const { return std::find_if(mImages.begin(), mImages.end(), [&](auto const& imageData) { diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.h b/filament/backend/src/vulkan/VulkanExternalImageManager.h index b458c1f3d241..0f3173fc5ebf 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.h +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.h @@ -65,8 +65,6 @@ class VulkanExternalImageManager { void bindExternallySampledTexture(fvkmemory::resource_ptr set, uint8_t bindingPoint, fvkmemory::resource_ptr image, SamplerParams samplerParams); - void bindStream(fvkmemory::resource_ptr set, uint8_t bindingPoint, - fvkmemory::resource_ptr stream, SamplerParams samplerParams); void clearTextureBinding(fvkmemory::resource_ptr set, uint8_t bindingPoint); @@ -77,11 +75,6 @@ class VulkanExternalImageManager { void removeExternallySampledTexture(fvkmemory::resource_ptr image); bool isExternallySampledTexture(fvkmemory::resource_ptr image) const; - bool isStreamedTexture(fvkmemory::resource_ptr image) const; - - // For a stream backed VulkanTexture, we are receiving new frames periodically, add them to the tracking system - void bindStreamFrame(fvkmemory::resource_ptr stream, - fvkmemory::resource_ptr frame); VkSamplerYcbcrConversion getVkSamplerYcbcrConversion( VulkanPlatform::ExternalImageMetadata const& metadata); @@ -93,11 +86,11 @@ class VulkanExternalImageManager { VkSamplerYcbcrConversion conversion = VK_NULL_HANDLE; }; - bool hasExternalSampler(fvkmemory::resource_ptr set, bool& streamed) const; + bool hasExternalSampler(fvkmemory::resource_ptr set) const; private: void updateSetAndLayout(fvkmemory::resource_ptr set, - fvkmemory::resource_ptr layout, bool streamed); + fvkmemory::resource_ptr layout); void updateImage(ImageData* imageData); @@ -117,16 +110,9 @@ class VulkanExternalImageManager { SamplerParams samplerParams; bool bound = false; }; - struct SetStreamBindingInfo { - uint8_t binding = 0; - fvkmemory::resource_ptr stream; - fvkmemory::resource_ptr set; - SamplerParams samplerParams; - }; // Use vectors instead of hash maps because we only expect small number of entries. std::vector mSetBindings; - std::vector mSetStreamBindings; std::vector mImages; }; diff --git a/filament/backend/src/vulkan/VulkanHandles.h b/filament/backend/src/vulkan/VulkanHandles.h index f905deb07621..a1d6dc03b829 100644 --- a/filament/backend/src/vulkan/VulkanHandles.h +++ b/filament/backend/src/vulkan/VulkanHandles.h @@ -185,11 +185,6 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { return mExternalSamplerVkSet; } - void setHasStreamedTexture(bool does) { mHasStreamedTexture = does; - } - - bool getHasStreamedTexture()const { return mHasStreamedTexture; } - void setExternalSamplerVkSet(VkDescriptorSet vkset, OnRecycle onRecycle) { mExternalSamplerVkSet = vkset; if (mOnRecycleExternalSamplerFn) { @@ -222,7 +217,6 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { private: VkDescriptorSet const mVkSet; VkDescriptorSet mExternalSamplerVkSet = VK_NULL_HANDLE; - bool mHasStreamedTexture = false; backend::DescriptorSetOffsetArray mOffsets; std::vector> mResources; From 3a1aebe5e93ef03dca3ced03ea25f9f8b5a07c82 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Thu, 28 Aug 2025 09:59:28 -0700 Subject: [PATCH 21/43] Fixed full screen bug. --- filament/backend/src/vulkan/VulkanDriver.cpp | 24 +++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 8f7cb050ee71..8911e7533a2e 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -386,9 +386,11 @@ void VulkanDriver::bindStreamedTexture(fvkmemory::resource_ptr set, uint8_t bindingPoint) { - std::erase_if(mStreamedTexturesBindings, [&](streamedTextureBinding& binding) { - return ((binding.set == set) && (binding.binding == bindingPoint)); - }); + auto iter = std::remove_if(mStreamedTexturesBindings.begin(), mStreamedTexturesBindings.end(), + [&](streamedTextureBinding& binding) { + return ((binding.set == set) && (binding.binding == bindingPoint)); + }); + mStreamedTexturesBindings.erase(iter, mStreamedTexturesBindings.end()); } void VulkanDriver::onStreamAcquireImage(fvkmemory::resource_ptr image, @@ -399,9 +401,19 @@ void VulkanDriver::onStreamAcquireImage(fvkmemory::resource_ptr i mExternalImageManager.bindExternallySampledTexture(data.set, data.binding, image, data.samplerParams); } else { - // just update the sampler set - mDescriptorSetCache.updateSamplerForExternalSamplerSet(data.set, data.binding, - image); + // For some reason, some of the frames coming to us, are on streams where the + // descriptor set isn't external... + if (data.set->getExternalSamplerVkSet()) { + mDescriptorSetCache.updateSamplerForExternalSamplerSet(data.set, data.binding, + image); + } else { + //... In this case we just default to using the normal path and update the sampler. + VulkanSamplerCache::Params cacheParams = { + .sampler = data.samplerParams, + }; + VkSampler const vksampler = mSamplerCache.getSampler(cacheParams); + mDescriptorSetCache.updateSampler(data.set, data.binding, image, vksampler); + } } } } From 0c07c3f67f94c43bd18c17717e0ca4d1cfcdfaa5 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Fri, 29 Aug 2025 10:24:35 -0700 Subject: [PATCH 22/43] Refactoring of the code. --- filament/backend/CMakeLists.txt | 2 + filament/backend/src/vulkan/VulkanDriver.cpp | 48 +---------- filament/backend/src/vulkan/VulkanDriver.h | 18 +--- .../src/vulkan/VulkanExternalImageManager.h | 2 +- .../src/vulkan/VulkanStreamedImageManager.cpp | 83 +++++++++++++++++++ .../src/vulkan/VulkanStreamedImageManager.h | 68 +++++++++++++++ 6 files changed, 160 insertions(+), 61 deletions(-) create mode 100644 filament/backend/src/vulkan/VulkanStreamedImageManager.cpp create mode 100644 filament/backend/src/vulkan/VulkanStreamedImageManager.h diff --git a/filament/backend/CMakeLists.txt b/filament/backend/CMakeLists.txt index 3796af3aef17..f881f99ddac5 100644 --- a/filament/backend/CMakeLists.txt +++ b/filament/backend/CMakeLists.txt @@ -199,6 +199,8 @@ if (FILAMENT_SUPPORTS_VULKAN) src/vulkan/VulkanDriverFactory.h src/vulkan/VulkanExternalImageManager.cpp src/vulkan/VulkanExternalImageManager.h + src/vulkan/VulkanStreamedImageManager.cpp + src/vulkan/VulkanStreamedImageManager.h src/vulkan/VulkanFboCache.cpp src/vulkan/VulkanFboCache.h src/vulkan/VulkanHandles.cpp diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 8911e7533a2e..f6648fb7917c 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -231,6 +231,7 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext& context, mQueryManager(mPlatform->getDevice()), mExternalImageManager(platform, &mSamplerCache, &mYcbcrConversionCache, &mDescriptorSetCache, &mDescriptorSetLayoutCache), + mStreamedImageManager(&mExternalImageManager, &mDescriptorSetCache, &mSamplerCache), mIsSRGBSwapChainSupported(mPlatform->getCustomization().isSRGBSwapChainSupported), mStereoscopicType(driverConfig.stereoscopicType) { @@ -378,47 +379,6 @@ void VulkanDriver::tick(int) { mCommands.updateFences(); } -void VulkanDriver::bindStreamedTexture(fvkmemory::resource_ptr set, - uint8_t bindingPoint, fvkmemory::resource_ptr image, - SamplerParams samplerParams) { - mStreamedTexturesBindings.push_back({ bindingPoint, image, set, samplerParams }); -} - -void VulkanDriver::unbindStreamedTexture(fvkmemory::resource_ptr set, - uint8_t bindingPoint) { - auto iter = std::remove_if(mStreamedTexturesBindings.begin(), mStreamedTexturesBindings.end(), - [&](streamedTextureBinding& binding) { - return ((binding.set == set) && (binding.binding == bindingPoint)); - }); - mStreamedTexturesBindings.erase(iter, mStreamedTexturesBindings.end()); -} - -void VulkanDriver::onStreamAcquireImage(fvkmemory::resource_ptr image, - fvkmemory::resource_ptr stream, bool newImage) { - for (streamedTextureBinding const& data: mStreamedTexturesBindings) { - if (data.image->getStream() == stream) { - if (newImage) { - mExternalImageManager.bindExternallySampledTexture(data.set, data.binding, image, - data.samplerParams); - } else { - // For some reason, some of the frames coming to us, are on streams where the - // descriptor set isn't external... - if (data.set->getExternalSamplerVkSet()) { - mDescriptorSetCache.updateSamplerForExternalSamplerSet(data.set, data.binding, - image); - } else { - //... In this case we just default to using the normal path and update the sampler. - VulkanSamplerCache::Params cacheParams = { - .sampler = data.samplerParams, - }; - VkSampler const vksampler = mSamplerCache.getSampler(cacheParams); - mDescriptorSetCache.updateSampler(data.set, data.binding, image, vksampler); - } - } - } - } -} - // Garbage collection should not occur too frequently, only about once per frame. Internally, the // eviction time of various resources is often measured in terms of an approximate frame number // rather than the wall clock, because we must wait 3 frames after a DriverAPI-level resource has @@ -497,7 +457,7 @@ void VulkanDriver::updateDescriptorSetTexture( mExternalImageManager.bindExternallySampledTexture(set, binding, texture, params); mAppState.hasBoundExternalImages = true; } else if (bool(texture->getStream())) { - bindStreamedTexture(set, binding, texture, params); + mStreamedImageManager.bindStreamedTexture(set, binding, texture, params); mAppState.hasBoundExternalImages = true; } else { @@ -507,7 +467,7 @@ void VulkanDriver::updateDescriptorSetTexture( VkSampler const vksampler = mSamplerCache.getSampler(cacheParams); mDescriptorSetCache.updateSampler(set, binding, texture, vksampler); mExternalImageManager.clearTextureBinding(set, binding); - unbindStreamedTexture(set, binding); + mStreamedImageManager.unbindStreamedTexture(set, binding); } } @@ -1131,7 +1091,7 @@ void VulkanDriver::updateStreams(CommandStream* driver) { texture = newTexture; } - onStreamAcquireImage(texture, stream, newImage); + mStreamedImageManager.onStreamAcquireImage(texture, stream, newImage); } mStreamsWithPendingAcquiredImage.clear(); } diff --git a/filament/backend/src/vulkan/VulkanDriver.h b/filament/backend/src/vulkan/VulkanDriver.h index 5b459ead992f..7715cc5dc7cc 100644 --- a/filament/backend/src/vulkan/VulkanDriver.h +++ b/filament/backend/src/vulkan/VulkanDriver.h @@ -33,6 +33,7 @@ #include "vulkan/VulkanDescriptorSetCache.h" #include "vulkan/VulkanDescriptorSetLayoutCache.h" #include "vulkan/VulkanExternalImageManager.h" +#include "vulkan/VulkanStreamedImageManager.h" #include "vulkan/VulkanPipelineLayoutCache.h" #include "vulkan/memory/ResourceManager.h" #include "vulkan/memory/ResourcePointer.h" @@ -121,13 +122,6 @@ class VulkanDriver final : public DriverBase { VulkanDriver& operator=(VulkanDriver const&) = delete; private: - void bindStreamedTexture(fvkmemory::resource_ptr set, uint8_t bindingPoint, - fvkmemory::resource_ptr image, SamplerParams samplerParams); - void unbindStreamedTexture(fvkmemory::resource_ptr set, - uint8_t bindingPoint); - void onStreamAcquireImage(fvkmemory::resource_ptr image, - fvkmemory::resource_ptr stream, bool newImage); - void collectGarbage(); void bindPipelineImpl(PipelineState const& pipelineState, VkPipelineLayout pipelineLayout, fvkutils::DescriptorSetMask descriptorSetMask); @@ -160,6 +154,7 @@ class VulkanDriver final : public DriverBase { VulkanDescriptorSetCache mDescriptorSetCache; VulkanQueryManager mQueryManager; VulkanExternalImageManager mExternalImageManager; + VulkanStreamedImageManager mStreamedImageManager; // This is necessary for us to write to push constants after binding a pipeline. using DescriptorSetLayoutHandleList = std::array, @@ -199,15 +194,6 @@ class VulkanDriver final : public DriverBase { // setAcquiredImage is a DECL_DRIVER_API_SYNCHRONOUS_N which means we don't necessarily have the // data to process it at call time. So we store it and process it during updateStreams. std::vector> mStreamsWithPendingAcquiredImage; - - struct streamedTextureBinding { - uint8_t binding = 0; - fvkmemory::resource_ptr image; - fvkmemory::resource_ptr set; - SamplerParams samplerParams; - }; - // keep track of all the stream bindings - std::vector mStreamedTexturesBindings; }; } // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.h b/filament/backend/src/vulkan/VulkanExternalImageManager.h index 0f3173fc5ebf..399361532a3d 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.h +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.h @@ -31,7 +31,7 @@ class VulkanSamplerCache; class VulkanDescriptorSetLayoutCache; class VulkanDescriptorSetCache; -// Manages the logic of external images and their quirks wrt Vulikan. +// Manages the logic of external images and their quirks wrt Vulkan. class VulkanExternalImageManager { public: diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp new file mode 100644 index 000000000000..62956ca3fccc --- /dev/null +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "VulkanStreamedImageManager.h" + +#include "VulkanDescriptorSetCache.h" +#include "VulkanExternalImageManager.h" +#include "VulkanDescriptorSetLayoutCache.h" +#include "VulkanSamplerCache.h" + +namespace filament::backend { + VulkanStreamedImageManager::VulkanStreamedImageManager( + VulkanExternalImageManager* manager, + VulkanDescriptorSetCache* descriptorSet, + VulkanSamplerCache* samplerCache) + : mExternalImageManager(manager), + mDescriptorSetCache(descriptorSet), + mSamplerCache(samplerCache) { + +} + +VulkanStreamedImageManager::~VulkanStreamedImageManager() = default; + +void VulkanStreamedImageManager::terminate() {} + +void VulkanStreamedImageManager::bindStreamedTexture( + fvkmemory::resource_ptr set, + uint8_t bindingPoint, fvkmemory::resource_ptr image, + SamplerParams samplerParams) { + mStreamedTexturesBindings.push_back({ bindingPoint, image, set, samplerParams }); +} + +void VulkanStreamedImageManager::unbindStreamedTexture( + fvkmemory::resource_ptr set, + uint8_t bindingPoint) { + auto iter = std::remove_if(mStreamedTexturesBindings.begin(), mStreamedTexturesBindings.end(), + [&](streamedTextureBinding& binding) { + return ((binding.set == set) && (binding.binding == bindingPoint)); + }); + mStreamedTexturesBindings.erase(iter, mStreamedTexturesBindings.end()); +} + +void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptr image, + fvkmemory::resource_ptr stream, bool newImage) { + for (streamedTextureBinding const& data: mStreamedTexturesBindings) { + if (data.image->getStream() == stream) { + if (newImage) { + mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image, + data.samplerParams); + } else { + // For some reason, some of the frames coming to us, are on streams where the + // descriptor set isn't external... + if (data.set->getExternalSamplerVkSet()) { + mDescriptorSetCache->updateSamplerForExternalSamplerSet(data.set, data.binding, + image); + } else { + //... In this case we just default to using the normal path and update the + //sampler. + VulkanSamplerCache::Params cacheParams = { + .sampler = data.samplerParams, + }; + VkSampler const vksampler = mSamplerCache->getSampler(cacheParams); + mDescriptorSetCache->updateSampler(data.set, data.binding, image, vksampler); + } + } + } + } +} + +} // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.h b/filament/backend/src/vulkan/VulkanStreamedImageManager.h new file mode 100644 index 000000000000..172ea5970e18 --- /dev/null +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TNT_FILAMENT_BACKEND_CACHING_VULKANSTREAMEDIMAGEMANAGER_H +#define TNT_FILAMENT_BACKEND_CACHING_VULKANSTREAMEDIMAGEMANAGER_H + +#include "VulkanHandles.h" + +#include + +#include +#include + +namespace filament::backend { + +class VulkanExternalImageManager; +class VulkanDescriptorSetCache; +class VulkanSamplerCache; + +// Manages the logic of streamed and streamed images. +class VulkanStreamedImageManager { +public: + VulkanStreamedImageManager( + VulkanExternalImageManager* manager, + VulkanDescriptorSetCache* descriptorSet, + VulkanSamplerCache* samplerCache); + ~VulkanStreamedImageManager(); + void terminate(); + +public: + void bindStreamedTexture(fvkmemory::resource_ptr set, uint8_t bindingPoint, + fvkmemory::resource_ptr image, SamplerParams samplerParams); + void unbindStreamedTexture(fvkmemory::resource_ptr set, + uint8_t bindingPoint); + void onStreamAcquireImage(fvkmemory::resource_ptr image, + fvkmemory::resource_ptr stream, bool newImage); + +private: + struct streamedTextureBinding { + uint8_t binding = 0; + fvkmemory::resource_ptr image; + fvkmemory::resource_ptr set; + SamplerParams samplerParams; + }; + // keep track of all the stream bindings + std::vector mStreamedTexturesBindings; + + VulkanExternalImageManager* mExternalImageManager; + VulkanDescriptorSetCache* mDescriptorSetCache; + VulkanSamplerCache* mSamplerCache; +}; + +} // namespace filament::backend + +#endif // TNT_FILAMENT_BACKEND_CACHING_VULKANSTREAMEDIMAGEMANAGER_H From 39be20b06091329d6371f3869036adb4ab44d03c Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Fri, 29 Aug 2025 10:33:36 -0700 Subject: [PATCH 23/43] Typo. --- filament/backend/src/vulkan/VulkanExternalImageManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index 886de2a0f22c..18ecd0264258 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -121,7 +121,7 @@ fvkutils::DescriptorSetMask VulkanExternalImageManager::prepareBindSets(LayoutAr } bool VulkanExternalImageManager::hasExternalSampler( - fvkmemory::resource_ptr set)const { + fvkmemory::resource_ptr set) const { auto itr = std::find_if(mSetBindings.begin(), mSetBindings.end(), [&](SetBindingInfo const& info) { return info.set == set; }); return itr != mSetBindings.end(); From db83dc057e88478c2a8a39ce49aa49e111e0afd7 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 3 Sep 2025 20:14:13 -0700 Subject: [PATCH 24/43] GitHub feedback. --- filament/backend/src/vulkan/VulkanDriver.cpp | 5 +++++ filament/backend/src/vulkan/VulkanStreamedImageManager.cpp | 6 +++++- filament/backend/src/vulkan/VulkanTexture.h | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index f6648fb7917c..dab88ee444df 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1002,6 +1002,7 @@ void VulkanDriver::destroyDescriptorSet(Handle dsh) { } Handle VulkanDriver::createStreamNative(void* nativeStream) { + FVK_SYSTRACE_SCOPE(); auto handle = mResourceManager.allocHandle(); auto stream = resource_ptr::make(&mResourceManager, handle); stream.inc(); @@ -1009,6 +1010,7 @@ Handle VulkanDriver::createStreamNative(void* nativeStream) { } Handle VulkanDriver::createStreamAcquired() { + FVK_SYSTRACE_SCOPE(); auto handle = mResourceManager.allocHandle(); auto stream = resource_ptr::make(&mResourceManager, handle); stream.inc(); @@ -1024,6 +1026,7 @@ void VulkanDriver::setAcquiredImage(Handle sh, void* image, const math } void VulkanDriver::setStreamDimensions(Handle sh, uint32_t width, uint32_t height) { + FVK_SYSTRACE_SCOPE(); auto stream = resource_ptr::cast(&mResourceManager, sh); stream->width = width; stream->height = height; @@ -1034,6 +1037,7 @@ int64_t VulkanDriver::getStreamTimestamp(Handle sh) { } void VulkanDriver::updateStreams(CommandStream* driver) { + FVK_SYSTRACE_SCOPE(); if (UTILS_UNLIKELY(!mStreamsWithPendingAcquiredImage.empty())) { for (auto& stream: mStreamsWithPendingAcquiredImage) { if (stream->previousNeedsRelease()) { @@ -1416,6 +1420,7 @@ TimerQueryResult VulkanDriver::getTimerQueryValue(Handle tqh, uint } void VulkanDriver::setExternalStream(Handle th, Handle sh) { + FVK_SYSTRACE_SCOPE(); auto t = resource_ptr::cast(&mResourceManager, th); assert_invariant(t); auto s = resource_ptr::cast(&mResourceManager, sh); diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp index 62956ca3fccc..4faed3f3a692 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp @@ -34,7 +34,7 @@ namespace filament::backend { VulkanStreamedImageManager::~VulkanStreamedImageManager() = default; -void VulkanStreamedImageManager::terminate() {} +void VulkanStreamedImageManager::terminate() { mStreamedTexturesBindings.clear(); } void VulkanStreamedImageManager::bindStreamedTexture( fvkmemory::resource_ptr set, @@ -64,6 +64,10 @@ void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptrgetExternalSamplerVkSet()) { + // Eventually the updateSampler and updateSamplerForExternalSamplerSet + // will call to vkUpdateDescriptorSets with a VkWriteDescriptorSet + // type VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER the + // VkDescriptorImageInfo will contain the view of the new image frame. mDescriptorSetCache->updateSamplerForExternalSamplerSet(data.set, data.binding, image); } else { diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index b87dad3122a5..22ad54a624f3 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -61,6 +61,7 @@ struct VulkanStream : public HwStream, fvkmemory::Resource { private: AcquiredImage mAcquired; AcquiredImage mPrevious; + // #TODO b/442937292 std::map> mTextures; }; From 74ffe4ebb156a41d46960c315b40d609cd58a046 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Thu, 4 Sep 2025 19:57:38 -0700 Subject: [PATCH 25/43] Github feedback --- filament/backend/src/vulkan/VulkanDriver.cpp | 161 ++++++++++++------ filament/backend/src/vulkan/VulkanDriver.h | 5 + .../src/vulkan/VulkanStreamedImageManager.h | 2 +- filament/backend/src/vulkan/VulkanTexture.h | 44 +++-- 4 files changed, 144 insertions(+), 68 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 51638433db31..f80dada344b7 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -459,8 +459,7 @@ void VulkanDriver::updateDescriptorSetTexture( } else if (bool(texture->getStream())) { mStreamedImageManager.bindStreamedTexture(set, binding, texture, params); mAppState.hasBoundExternalImages = true; - } - else { + } else { VulkanSamplerCache::Params cacheParams = { .sampler = params, }; @@ -1025,11 +1024,8 @@ void VulkanDriver::destroyDescriptorSet(Handle dsh) { } Handle VulkanDriver::createStreamNative(void* nativeStream, utils::CString tag) { - FVK_SYSTRACE_SCOPE(); - auto handle = mResourceManager.allocHandle(); - auto stream = resource_ptr::make(&mResourceManager, handle); - stream.inc(); - return handle; + FILAMENT_CHECK_PRECONDITION(false) << "createStreamNative not supported in Vulkan."; + return {}; } Handle VulkanDriver::createStreamAcquired(utils::CString tag) { @@ -1044,7 +1040,8 @@ void VulkanDriver::setAcquiredImage(Handle sh, void* image, const math CallbackHandler* handler, StreamCallback cb, void* userData) { FVK_SYSTRACE_SCOPE(); auto stream = resource_ptr::cast(&mResourceManager, sh); - stream->acquire({ image, cb, userData, handler }); + stream->acquire({ image, cb, userData, handler}); + stream->setTransform(transform); mStreamsWithPendingAcquiredImage.push_back(stream); } @@ -1059,6 +1056,18 @@ int64_t VulkanDriver::getStreamTimestamp(Handle sh) { return 0; } +math::mat3f VulkanDriver::getStreamTransformMatrix(Handle sh) { + if (sh) { + auto stream = resource_ptr::cast(&mResourceManager, sh); + if (stream->streamType == StreamType::NATIVE) { + FILAMENT_CHECK_PRECONDITION(false) << "Native Stream not supported in Vulkan."; + } else { + return stream->getTransform(); + } + } + return math::mat3f(); +} + void VulkanDriver::updateStreams(CommandStream* driver) { FVK_SYSTRACE_SCOPE(); if (UTILS_UNLIKELY(!mStreamsWithPendingAcquiredImage.empty())) { @@ -1067,58 +1076,65 @@ void VulkanDriver::updateStreams(CommandStream* driver) { scheduleRelease(stream->takePrevious()); } - auto texture = stream->getTexture(stream->getAcquired().image); - bool newImage = false; - if (!texture) { - auto externalImage = - mPlatform->createExternalImageFromRaw(stream->getAcquired().image, false); - auto metadata = mPlatform->extractExternalImageMetadata(externalImage); - auto imgData = mPlatform->createVkImageFromExternal(externalImage); - - assert_invariant(imgData.internal.valid() || imgData.external.valid()); - - VkFormat vkformat = metadata.format; - VkImage vkimage = VK_NULL_HANDLE; - VkDeviceMemory memory = VK_NULL_HANDLE; - if (imgData.internal.valid()) { - metadata.externalFormat = 0; - vkimage = imgData.internal.image; - memory = imgData.internal.memory; - } else { - vkformat = VK_FORMAT_UNDEFINED; - vkimage = imgData.external.image; - memory = imgData.external.memory; - } - - VkSamplerYcbcrConversion const conversion = - mExternalImageManager.getVkSamplerYcbcrConversion(metadata); + // This executes on the backend thread (updateStreams is synchonous which means it + // executes on the user thread) Note: stream is captured by copy which is fine, this is + // a copy of a resource_ptr. We only need it find the associated stream + // inside the mStreamedImageManager texture bindings + driver->queueCommand([this, stream, s = stream.get(), + image = stream->getAcquired().image, + transform = stream->getTransform()]() { + auto texture = s->getTexture(image); + bool newImage = false; + if (!texture) { + auto externalImage = mPlatform->createExternalImageFromRaw(image, false); + auto metadata = mPlatform->extractExternalImageMetadata(externalImage); + auto imgData = mPlatform->createVkImageFromExternal(externalImage); + + assert_invariant(imgData.internal.valid() || imgData.external.valid()); + + VkFormat vkformat = metadata.format; + VkImage vkimage = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + if (imgData.internal.valid()) { + metadata.externalFormat = 0; + vkimage = imgData.internal.image; + memory = imgData.internal.memory; + } else { + vkformat = VK_FORMAT_UNDEFINED; + vkimage = imgData.external.image; + memory = imgData.external.memory; + } + + VkSamplerYcbcrConversion const conversion = + mExternalImageManager.getVkSamplerYcbcrConversion(metadata); + + auto newTexture = resource_ptr::construct(&mResourceManager, + mContext, mPlatform->getDevice(), mAllocator, &mResourceManager, + &mCommands, vkimage, memory, vkformat, conversion, metadata.samples, + metadata.width, metadata.height, metadata.layers, + metadata.filamentUsage, mStagePool); - auto newTexture = resource_ptr::construct(&mResourceManager, mContext, - mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, vkimage, - memory, vkformat, conversion, metadata.samples, metadata.width, - metadata.height, metadata.layers, metadata.filamentUsage, mStagePool); - - // We shouldn't need to do this when we allocate the texture - if (true) { auto& commands = mCommands.get(); // Unlike uploaded textures or swapchains, we need to explicit transition this // texture into the read layout. newTexture->transitionLayout(&commands, newTexture->getPrimaryViewRange(), VulkanLayout::FRAG_READ); - } - if (imgData.external.valid()) { - mExternalImageManager.addExternallySampledTexture(newTexture, externalImage); - // Cache the AHB backed image. - stream->pushImage(stream->getAcquired().image, newTexture); - newImage = true; - } + if (imgData.external.valid()) { + mExternalImageManager.addExternallySampledTexture(newTexture, + externalImage); + // Cache the AHB backed image. + s->pushImage(image, newTexture); + newImage = true; + } - newTexture.inc(); - texture = newTexture; - } + newTexture.inc(); + texture = newTexture; + } - mStreamedImageManager.onStreamAcquireImage(texture, stream, newImage); + // Note that we capture the + mStreamedImageManager.onStreamAcquireImage(texture, stream, newImage); + }); } mStreamsWithPendingAcquiredImage.clear(); } @@ -1367,7 +1383,34 @@ void VulkanDriver::updateIndexBuffer(Handle ibh, BufferDescriptor void VulkanDriver::registerBufferObjectStreams(Handle boh, BufferObjectStreamDescriptor&& streams) { - // Noop + + auto bo = resource_ptr::cast(&mResourceManager, boh); + mStreamUniformDescriptors[bo->getVkBuffer()] = std::move(streams); +} + +// This needs to be discussed because Vulkan has different default Matrix formats +// right now this is following GL. +static void copyMat3f(void* addr, size_t const offset, const math::mat3f& v) noexcept { + struct mat43 { + float v[3][4]; + }; + + addr = static_cast(addr) + offset; + mat43& temp = *static_cast(addr); + + temp.v[0][0] = v[0][0]; + temp.v[0][1] = v[0][1]; + temp.v[0][2] = v[0][2]; + + temp.v[1][0] = v[1][0]; + temp.v[1][1] = v[1][1]; + temp.v[1][2] = v[1][2]; + + temp.v[2][0] = v[2][0]; + temp.v[2][1] = v[2][1]; + temp.v[2][2] = v[2][2]; + + // don't store anything in temp.v[][3] because there could be uniforms packed there } void VulkanDriver::updateBufferObject(Handle boh, BufferDescriptor&& bd, @@ -1378,6 +1421,20 @@ void VulkanDriver::updateBufferObject(Handle boh, BufferDescript commands.acquire(bo); bo->loadFromCpu(commands, bd.buffer, byteOffset, bd.size); + if (UTILS_UNLIKELY(!mStreamUniformDescriptors.empty())) { + auto streamDescriptors = mStreamUniformDescriptors.find(bo->getVkBuffer()); + if (streamDescriptors != mStreamUniformDescriptors.end()) { + for (auto const& [offset, stream, associationType]: + streamDescriptors->second.mStreams) { + if (associationType == BufferObjectStreamAssociationType::TRANSFORM_MATRIX) { + auto transform = getStreamTransformMatrix(stream); + copyMat3f(bd.buffer, offset, transform); + } + } + mStreamUniformDescriptors.erase(streamDescriptors); + } + } + scheduleDestroy(std::move(bd)); } diff --git a/filament/backend/src/vulkan/VulkanDriver.h b/filament/backend/src/vulkan/VulkanDriver.h index 7715cc5dc7cc..ed2bb8880e88 100644 --- a/filament/backend/src/vulkan/VulkanDriver.h +++ b/filament/backend/src/vulkan/VulkanDriver.h @@ -156,6 +156,11 @@ class VulkanDriver final : public DriverBase { VulkanExternalImageManager mExternalImageManager; VulkanStreamedImageManager mStreamedImageManager; + // Stream transforms + std::unordered_map mStreamUniformDescriptors; + math::mat3f getStreamTransformMatrix(Handle sh); + + // This is necessary for us to write to push constants after binding a pipeline. using DescriptorSetLayoutHandleList = std::array, VulkanDescriptorSetLayout::UNIQUE_DESCRIPTOR_SET_COUNT>; diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.h b/filament/backend/src/vulkan/VulkanStreamedImageManager.h index 172ea5970e18..76036aad8b20 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.h +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.h @@ -30,7 +30,7 @@ class VulkanExternalImageManager; class VulkanDescriptorSetCache; class VulkanSamplerCache; -// Manages the logic of streamed and streamed images. +// Manages the logic of streamed images. class VulkanStreamedImageManager { public: VulkanStreamedImageManager( diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index 22ad54a624f3..7e1691ac33b7 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -37,30 +37,41 @@ namespace filament::backend { struct VulkanTexture; struct VulkanStream : public HwStream, fvkmemory::Resource { - fvkmemory::resource_ptr getTexture(void* ahb) { - fvkmemory::resource_ptr frame; - const auto& it = mTextures.find(ahb); - if (it != mTextures.end()) frame = it->second; - return frame; - } - void pushImage(void* ahb, fvkmemory::resource_ptr tex) { mTextures[ahb] = tex; } + + //-- These are only called from the frontend void acquire(const AcquiredImage& image) { - mPrevious = mAcquired; - mAcquired = image; + user_thread.mPrevious = user_thread.mAcquired; + user_thread.mAcquired = image; } - bool previousNeedsRelease() const { return (mPrevious.image != nullptr); } + bool previousNeedsRelease() const { return (user_thread.mPrevious.image != nullptr); } // this function will null the previous once the caller takes it. // It ensures we don't schedule for release twice. AcquiredImage takePrevious() { - AcquiredImage previous = mPrevious; - mPrevious = { nullptr, nullptr, nullptr, nullptr }; + AcquiredImage previous = user_thread.mPrevious; + user_thread.mPrevious = { nullptr, nullptr, nullptr, nullptr }; return previous; } - const AcquiredImage& getAcquired() const { return mAcquired; } + const AcquiredImage& getAcquired() const { return user_thread.mAcquired; } + const math::mat3f& getTransform() const { return user_thread.mTransform; } + void setTransform(const math::mat3f& transform) { user_thread.mTransform = transform; } + + //-- This methods are only called from the backend thread + fvkmemory::resource_ptr getTexture(void* ahb) { + fvkmemory::resource_ptr frame; + const auto& it = mTextures.find(ahb); + if (it != mTextures.end()) frame = it->second; + return frame; + } + void pushImage(void* ahb, fvkmemory::resource_ptr tex) { mTextures[ahb] = tex; } private: + // These are only called from the frontend + struct { AcquiredImage mAcquired; AcquiredImage mPrevious; + math::mat3f mTransform; + } user_thread; + // #TODO b/442937292 std::map> mTextures; }; @@ -214,10 +225,13 @@ struct VulkanTexture : public HwTexture, fvkmemory::Resource { return mState->mSidecarMSAA; } - void setStream(fvkmemory::resource_ptr stream) { mState->mStream = stream; + void setStream(fvkmemory::resource_ptr stream) { + mState->mStream = stream; } - fvkmemory::resource_ptr getStream() const { return mState->mStream; } + fvkmemory::resource_ptr getStream() const { + return mState->mStream; + } bool isTransientAttachment() const { return mState->mUsage & VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT; From 32ca1d3a77efa28098090a486be7336580e3dd76 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Thu, 4 Sep 2025 19:59:59 -0700 Subject: [PATCH 26/43] feedback. --- filament/backend/src/vulkan/VulkanStreamedImageManager.cpp | 4 ++-- filament/backend/src/vulkan/VulkanStreamedImageManager.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp index 4faed3f3a692..b409db37801a 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp @@ -47,7 +47,7 @@ void VulkanStreamedImageManager::unbindStreamedTexture( fvkmemory::resource_ptr set, uint8_t bindingPoint) { auto iter = std::remove_if(mStreamedTexturesBindings.begin(), mStreamedTexturesBindings.end(), - [&](streamedTextureBinding& binding) { + [&](StreamedTextureBinding& binding) { return ((binding.set == set) && (binding.binding == bindingPoint)); }); mStreamedTexturesBindings.erase(iter, mStreamedTexturesBindings.end()); @@ -55,7 +55,7 @@ void VulkanStreamedImageManager::unbindStreamedTexture( void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptr image, fvkmemory::resource_ptr stream, bool newImage) { - for (streamedTextureBinding const& data: mStreamedTexturesBindings) { + for (StreamedTextureBinding const& data: mStreamedTexturesBindings) { if (data.image->getStream() == stream) { if (newImage) { mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image, diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.h b/filament/backend/src/vulkan/VulkanStreamedImageManager.h index 76036aad8b20..365934eb2bc7 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.h +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.h @@ -49,14 +49,14 @@ class VulkanStreamedImageManager { fvkmemory::resource_ptr stream, bool newImage); private: - struct streamedTextureBinding { + struct StreamedTextureBinding { uint8_t binding = 0; fvkmemory::resource_ptr image; fvkmemory::resource_ptr set; SamplerParams samplerParams; }; // keep track of all the stream bindings - std::vector mStreamedTexturesBindings; + std::vector mStreamedTexturesBindings; VulkanExternalImageManager* mExternalImageManager; VulkanDescriptorSetCache* mDescriptorSetCache; From 34577000caf3931c89acff86a8343988ca5aecd8 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Fri, 5 Sep 2025 10:36:07 -0700 Subject: [PATCH 27/43] Typo --- filament/backend/src/vulkan/VulkanTexture.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index 7e1691ac33b7..e3eecff64a37 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -38,7 +38,7 @@ struct VulkanTexture; struct VulkanStream : public HwStream, fvkmemory::Resource { - //-- These are only called from the frontend + //-- These methods are only called from the frontend void acquire(const AcquiredImage& image) { user_thread.mPrevious = user_thread.mAcquired; user_thread.mAcquired = image; @@ -55,7 +55,7 @@ struct VulkanStream : public HwStream, fvkmemory::Resource { const math::mat3f& getTransform() const { return user_thread.mTransform; } void setTransform(const math::mat3f& transform) { user_thread.mTransform = transform; } - //-- This methods are only called from the backend thread + //-- These methods are only called from the backend thread fvkmemory::resource_ptr getTexture(void* ahb) { fvkmemory::resource_ptr frame; const auto& it = mTextures.find(ahb); From 85c899afa57ec74934e4f3ef797591b5f429790e Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Tue, 9 Sep 2025 08:16:58 -0700 Subject: [PATCH 28/43] Feedback. --- filament/backend/src/vulkan/VulkanDriver.cpp | 4 ++-- filament/backend/src/vulkan/VulkanDriver.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index f80dada344b7..34889fa6dde2 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1385,7 +1385,7 @@ void VulkanDriver::registerBufferObjectStreams(Handle boh, BufferObjectStreamDescriptor&& streams) { auto bo = resource_ptr::cast(&mResourceManager, boh); - mStreamUniformDescriptors[bo->getVkBuffer()] = std::move(streams); + mStreamUniformDescriptors[bo.get()] = std::move(streams); } // This needs to be discussed because Vulkan has different default Matrix formats @@ -1422,7 +1422,7 @@ void VulkanDriver::updateBufferObject(Handle boh, BufferDescript bo->loadFromCpu(commands, bd.buffer, byteOffset, bd.size); if (UTILS_UNLIKELY(!mStreamUniformDescriptors.empty())) { - auto streamDescriptors = mStreamUniformDescriptors.find(bo->getVkBuffer()); + auto streamDescriptors = mStreamUniformDescriptors.find(bo.get()); if (streamDescriptors != mStreamUniformDescriptors.end()) { for (auto const& [offset, stream, associationType]: streamDescriptors->second.mStreams) { diff --git a/filament/backend/src/vulkan/VulkanDriver.h b/filament/backend/src/vulkan/VulkanDriver.h index ed2bb8880e88..7c0499621670 100644 --- a/filament/backend/src/vulkan/VulkanDriver.h +++ b/filament/backend/src/vulkan/VulkanDriver.h @@ -157,7 +157,7 @@ class VulkanDriver final : public DriverBase { VulkanStreamedImageManager mStreamedImageManager; // Stream transforms - std::unordered_map mStreamUniformDescriptors; + std::unordered_map mStreamUniformDescriptors; math::mat3f getStreamTransformMatrix(Handle sh); From 0ba9c2ad76db5860de1f1aa83faacacc93e694db Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Tue, 9 Sep 2025 08:36:00 -0700 Subject: [PATCH 29/43] Feedback. --- .../src/vulkan/VulkanStreamedImageManager.cpp | 42 +++++++++---------- filament/backend/src/vulkan/VulkanTexture.h | 4 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp index b409db37801a..08bbbc955d6b 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp @@ -56,29 +56,29 @@ void VulkanStreamedImageManager::unbindStreamedTexture( void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptr image, fvkmemory::resource_ptr stream, bool newImage) { for (StreamedTextureBinding const& data: mStreamedTexturesBindings) { + // Find the right stream if (data.image->getStream() == stream) { - if (newImage) { - mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image, - data.samplerParams); - } else { - // For some reason, some of the frames coming to us, are on streams where the - // descriptor set isn't external... - if (data.set->getExternalSamplerVkSet()) { - // Eventually the updateSampler and updateSamplerForExternalSamplerSet - // will call to vkUpdateDescriptorSets with a VkWriteDescriptorSet - // type VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER the - // VkDescriptorImageInfo will contain the view of the new image frame. - mDescriptorSetCache->updateSamplerForExternalSamplerSet(data.set, data.binding, - image); - } else { - //... In this case we just default to using the normal path and update the - //sampler. - VulkanSamplerCache::Params cacheParams = { - .sampler = data.samplerParams, - }; - VkSampler const vksampler = mSamplerCache->getSampler(cacheParams); - mDescriptorSetCache->updateSampler(data.set, data.binding, image, vksampler); + // For some reason, some of the frames coming to us, are on streams where the + // descriptor set isn't external... + if (data.set->getExternalSamplerVkSet()) { + if (newImage) { + mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, + image, data.samplerParams); } + // Eventually the updateSampler and updateSamplerForExternalSamplerSet + // will call to vkUpdateDescriptorSets with a VkWriteDescriptorSet + // type VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER the + // VkDescriptorImageInfo will contain the view of the new image frame. + mDescriptorSetCache->updateSamplerForExternalSamplerSet(data.set, data.binding, + image); + } else { + //... In this case we just default to using the normal path and update the + // sampler. + VulkanSamplerCache::Params cacheParams = { + .sampler = data.samplerParams, + }; + VkSampler const vksampler = mSamplerCache->getSampler(cacheParams); + mDescriptorSetCache->updateSampler(data.set, data.binding, image, vksampler); } } } diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index e3eecff64a37..100378d8822f 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -48,7 +48,7 @@ struct VulkanStream : public HwStream, fvkmemory::Resource { // It ensures we don't schedule for release twice. AcquiredImage takePrevious() { AcquiredImage previous = user_thread.mPrevious; - user_thread.mPrevious = { nullptr, nullptr, nullptr, nullptr }; + user_thread.mPrevious = {}; return previous; } const AcquiredImage& getAcquired() const { return user_thread.mAcquired; } @@ -114,7 +114,7 @@ struct VulkanTextureState : public fvkmemory::Resource { // The texture with the sidecar owns the sidecar. fvkmemory::resource_ptr mSidecarMSAA; - // The stream this texture is associated with (I think cleaner than HwTexture::hwStream). + // The stream this texture is associated with. fvkmemory::resource_ptr mStream; VkImage const mTextureImage; From 78d5e34bdb97682d6266870e86c6399ed0ccf75c Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Thu, 11 Sep 2025 12:34:43 -0700 Subject: [PATCH 30/43] Feedback --- filament/backend/src/vulkan/VulkanDriver.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 34889fa6dde2..5a9a5c4fbee7 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1123,12 +1123,11 @@ void VulkanDriver::updateStreams(CommandStream* driver) { if (imgData.external.valid()) { mExternalImageManager.addExternallySampledTexture(newTexture, externalImage); - // Cache the AHB backed image. + // Cache the AHB backed image. Acquires the image here. s->pushImage(image, newTexture); newImage = true; } - newTexture.inc(); texture = newTexture; } From 1216024ef62ac44c75cfbadc0cd5ec79090f182b Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Thu, 11 Sep 2025 17:29:21 -0700 Subject: [PATCH 31/43] Feedback --- filament/backend/src/vulkan/VulkanStreamedImageManager.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp index 08bbbc955d6b..7cf3a0b745a3 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp @@ -65,12 +65,6 @@ void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptrbindExternallySampledTexture(data.set, data.binding, image, data.samplerParams); } - // Eventually the updateSampler and updateSamplerForExternalSamplerSet - // will call to vkUpdateDescriptorSets with a VkWriteDescriptorSet - // type VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER the - // VkDescriptorImageInfo will contain the view of the new image frame. - mDescriptorSetCache->updateSamplerForExternalSamplerSet(data.set, data.binding, - image); } else { //... In this case we just default to using the normal path and update the // sampler. From e022396f686a0590314fcfc6382369124ca35d9e Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Mon, 15 Sep 2025 11:29:37 -0700 Subject: [PATCH 32/43] Feedback --- .../include/backend/platforms/VulkanPlatform.h | 3 --- .../backend/platforms/VulkanPlatformAndroid.h | 2 -- filament/backend/src/vulkan/VulkanDriver.cpp | 12 ++++++++---- filament/backend/src/vulkan/VulkanTexture.h | 7 +++++-- .../src/vulkan/platform/VulkanPlatformAndroid.cpp | 5 ----- filament/backend/src/vulkan/utils/Image.cpp | 9 +++++++++ filament/backend/src/vulkan/utils/Image.h | 1 + 7 files changed, 23 insertions(+), 16 deletions(-) diff --git a/filament/backend/include/backend/platforms/VulkanPlatform.h b/filament/backend/include/backend/platforms/VulkanPlatform.h index 24616581097c..b708d8acf766 100644 --- a/filament/backend/include/backend/platforms/VulkanPlatform.h +++ b/filament/backend/include/backend/platforms/VulkanPlatform.h @@ -406,9 +406,6 @@ class VulkanPlatform : public Platform, utils::PrivateImplementation #include @@ -1041,7 +1042,7 @@ void VulkanDriver::setAcquiredImage(Handle sh, void* image, const math FVK_SYSTRACE_SCOPE(); auto stream = resource_ptr::cast(&mResourceManager, sh); stream->acquire({ image, cb, userData, handler}); - stream->setTransform(transform); + stream->setFrontEndTransform(transform); mStreamsWithPendingAcquiredImage.push_back(stream); } @@ -1062,7 +1063,9 @@ math::mat3f VulkanDriver::getStreamTransformMatrix(Handle sh) { if (stream->streamType == StreamType::NATIVE) { FILAMENT_CHECK_PRECONDITION(false) << "Native Stream not supported in Vulkan."; } else { - return stream->getTransform(); + // Backend call since getStreamTransformMatrix is called from async + // updateBufferObject. + return stream->getBackEndTransform(); } } return math::mat3f(); @@ -1082,11 +1085,12 @@ void VulkanDriver::updateStreams(CommandStream* driver) { // inside the mStreamedImageManager texture bindings driver->queueCommand([this, stream, s = stream.get(), image = stream->getAcquired().image, - transform = stream->getTransform()]() { + transform = stream->getFrontEndTransform()]() { auto texture = s->getTexture(image); + s->setBackendTransform(transform); bool newImage = false; if (!texture) { - auto externalImage = mPlatform->createExternalImageFromRaw(image, false); + auto externalImage = fvkutils::createExternalImageFromRaw(image, false); auto metadata = mPlatform->extractExternalImageMetadata(externalImage); auto imgData = mPlatform->createVkImageFromExternal(externalImage); diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index 100378d8822f..1f942cf3898f 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -52,10 +52,12 @@ struct VulkanStream : public HwStream, fvkmemory::Resource { return previous; } const AcquiredImage& getAcquired() const { return user_thread.mAcquired; } - const math::mat3f& getTransform() const { return user_thread.mTransform; } - void setTransform(const math::mat3f& transform) { user_thread.mTransform = transform; } + void setFrontEndTransform(const math::mat3f& transform) { user_thread.mTransform = transform; } + const math::mat3f& getFrontEndTransform() const { return user_thread.mTransform; } //-- These methods are only called from the backend thread + void setBackendTransform(const math::mat3f& transform) { mTransform = transform; } + const math::mat3f& getBackEndTransform() const { return mTransform; } fvkmemory::resource_ptr getTexture(void* ahb) { fvkmemory::resource_ptr frame; const auto& it = mTextures.find(ahb); @@ -73,6 +75,7 @@ struct VulkanStream : public HwStream, fvkmemory::Resource { } user_thread; // #TODO b/442937292 + math::mat3f mTransform; std::map> mTextures; }; diff --git a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp index ff7982188b0b..899891d090fc 100644 --- a/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp +++ b/filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp @@ -165,11 +165,6 @@ VulkanPlatformAndroid::ExternalImageVulkanAndroid::~ExternalImageVulkanAndroid() } } -Platform::ExternalImageHandle VulkanPlatformAndroid::createExternalImageFromRaw(void* image, - bool sRGB) noexcept { - return createExternalImage(reinterpret_cast(image), sRGB); -} - Platform::ExternalImageHandle VulkanPlatformAndroid::createExternalImage( AHardwareBuffer const* buffer, bool sRGB) noexcept { if (__builtin_available(android 26, *)) { diff --git a/filament/backend/src/vulkan/utils/Image.cpp b/filament/backend/src/vulkan/utils/Image.cpp index 5bea3009c741..f253592324b2 100644 --- a/filament/backend/src/vulkan/utils/Image.cpp +++ b/filament/backend/src/vulkan/utils/Image.cpp @@ -234,6 +234,15 @@ uint8_t reduceSampleCount(uint8_t sampleCount, VkSampleCountFlags mask) { return mostSignificantBit((sampleCount - 1) & mask); } +Platform::ExternalImageHandle createExternalImageFromRaw(void* image, + bool sRGB) { +#if defined(__ANDROID__) + return createExternalImage(reinterpret_cast(image), sRGB); +#else + return {}; +#endif +} + } // namespace filament::backend::fvkutils bool operator<(const VkImageSubresourceRange& a, const VkImageSubresourceRange& b) { diff --git a/filament/backend/src/vulkan/utils/Image.h b/filament/backend/src/vulkan/utils/Image.h index 5be3cd3769ab..82fa244ed057 100644 --- a/filament/backend/src/vulkan/utils/Image.h +++ b/filament/backend/src/vulkan/utils/Image.h @@ -106,6 +106,7 @@ VkImageAspectFlags getImageAspect(VkFormat format); uint8_t reduceSampleCount(uint8_t sampleCount, VkSampleCountFlags mask); +Platform::ExternalImageHandle createExternalImageFromRaw(void* image, bool sRGB); } // namespace fvkutils } // namespace filament::backend From 48cd99a24bf5c725bb181eea81516d52428dee42 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Thu, 18 Sep 2025 09:56:22 -0700 Subject: [PATCH 33/43] Fixing the Android path. --- filament/backend/src/vulkan/VulkanDriver.cpp | 3 ++- filament/backend/src/vulkan/VulkanTexture.h | 8 ++++---- filament/backend/src/vulkan/utils/Image.cpp | 19 ++++++++++++++----- filament/backend/src/vulkan/utils/Image.h | 5 ++++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index eedbe924ec60..8a2644917112 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -994,6 +994,7 @@ void VulkanDriver::destroySwapChain(Handle sch) { } void VulkanDriver::destroyStream(Handle sh) { + FVK_SYSTRACE_SCOPE(); if (!sh) { return; } @@ -1090,7 +1091,7 @@ void VulkanDriver::updateStreams(CommandStream* driver) { s->setBackendTransform(transform); bool newImage = false; if (!texture) { - auto externalImage = fvkutils::createExternalImageFromRaw(image, false); + auto externalImage = fvkutils::createExternalImageFromRaw(mPlatform, image, false); auto metadata = mPlatform->extractExternalImageMetadata(externalImage); auto imgData = mPlatform->createVkImageFromExternal(externalImage); diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index 1f942cf3898f..1991013ff644 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -69,14 +69,14 @@ struct VulkanStream : public HwStream, fvkmemory::Resource { private: // These are only called from the frontend struct { - AcquiredImage mAcquired; - AcquiredImage mPrevious; - math::mat3f mTransform; + AcquiredImage mAcquired; + AcquiredImage mPrevious; + math::mat3f mTransform; } user_thread; // #TODO b/442937292 math::mat3f mTransform; - std::map> mTextures; + std::unordered_map> mTextures; }; struct VulkanTextureState : public fvkmemory::Resource { diff --git a/filament/backend/src/vulkan/utils/Image.cpp b/filament/backend/src/vulkan/utils/Image.cpp index f253592324b2..0f195c68262c 100644 --- a/filament/backend/src/vulkan/utils/Image.cpp +++ b/filament/backend/src/vulkan/utils/Image.cpp @@ -18,6 +18,11 @@ #include "vulkan/VulkanTexture.h" +#if defined(__ANDROID__) +#include +#include +#endif + #include #include #include @@ -234,13 +239,17 @@ uint8_t reduceSampleCount(uint8_t sampleCount, VkSampleCountFlags mask) { return mostSignificantBit((sampleCount - 1) & mask); } -Platform::ExternalImageHandle createExternalImageFromRaw(void* image, - bool sRGB) { +filament::backend::Platform::ExternalImageHandle createExternalImageFromRaw( + filament::backend::VulkanPlatform* platform, void* image, bool sRGB) { + #if defined(__ANDROID__) - return createExternalImage(reinterpret_cast(image), sRGB); -#else - return {}; + if (__builtin_available(android 26, *)) { + return static_cast(platform) + ->createExternalImage(reinterpret_cast(image), sRGB); + } #endif + return {}; + } } // namespace filament::backend::fvkutils diff --git a/filament/backend/src/vulkan/utils/Image.h b/filament/backend/src/vulkan/utils/Image.h index 82fa244ed057..19762a8bee22 100644 --- a/filament/backend/src/vulkan/utils/Image.h +++ b/filament/backend/src/vulkan/utils/Image.h @@ -18,6 +18,7 @@ #define TNT_FILAMENT_BACKEND_VULKAN_UTILS_IMAGE_H #include +#include #include @@ -106,7 +107,9 @@ VkImageAspectFlags getImageAspect(VkFormat format); uint8_t reduceSampleCount(uint8_t sampleCount, VkSampleCountFlags mask); -Platform::ExternalImageHandle createExternalImageFromRaw(void* image, bool sRGB); +Platform::ExternalImageHandle createExternalImageFromRaw(filament::backend::VulkanPlatform* platform, + void* image, + bool sRGB); } // namespace fvkutils } // namespace filament::backend From 83a89b5dcd04b47f56ecd1070c07499559f2dd80 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Thu, 18 Sep 2025 10:58:35 -0700 Subject: [PATCH 34/43] Fixing the Android path. --- filament/backend/src/vulkan/utils/Image.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/filament/backend/src/vulkan/utils/Image.cpp b/filament/backend/src/vulkan/utils/Image.cpp index 0f195c68262c..b613bc95bc30 100644 --- a/filament/backend/src/vulkan/utils/Image.cpp +++ b/filament/backend/src/vulkan/utils/Image.cpp @@ -19,6 +19,7 @@ #include "vulkan/VulkanTexture.h" #if defined(__ANDROID__) +#include "backend/platforms/VulkanPlatformAndroid.h" #include #include #endif From 1a390f488b4fea8c8c4bdcfd2f733d57acab1f14 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Fri, 19 Sep 2025 07:46:35 -0700 Subject: [PATCH 35/43] Feedback. --- filament/backend/src/vulkan/VulkanDriver.cpp | 35 +++++++++---------- .../src/vulkan/VulkanExternalImageManager.cpp | 13 ++++++- .../src/vulkan/VulkanStreamedImageManager.cpp | 8 ++--- .../src/vulkan/VulkanStreamedImageManager.h | 2 +- 4 files changed, 32 insertions(+), 26 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 8a2644917112..1e747140789f 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1058,20 +1058,6 @@ int64_t VulkanDriver::getStreamTimestamp(Handle sh) { return 0; } -math::mat3f VulkanDriver::getStreamTransformMatrix(Handle sh) { - if (sh) { - auto stream = resource_ptr::cast(&mResourceManager, sh); - if (stream->streamType == StreamType::NATIVE) { - FILAMENT_CHECK_PRECONDITION(false) << "Native Stream not supported in Vulkan."; - } else { - // Backend call since getStreamTransformMatrix is called from async - // updateBufferObject. - return stream->getBackEndTransform(); - } - } - return math::mat3f(); -} - void VulkanDriver::updateStreams(CommandStream* driver) { FVK_SYSTRACE_SCOPE(); if (UTILS_UNLIKELY(!mStreamsWithPendingAcquiredImage.empty())) { @@ -1089,7 +1075,6 @@ void VulkanDriver::updateStreams(CommandStream* driver) { transform = stream->getFrontEndTransform()]() { auto texture = s->getTexture(image); s->setBackendTransform(transform); - bool newImage = false; if (!texture) { auto externalImage = fvkutils::createExternalImageFromRaw(mPlatform, image, false); auto metadata = mPlatform->extractExternalImageMetadata(externalImage); @@ -1130,14 +1115,13 @@ void VulkanDriver::updateStreams(CommandStream* driver) { externalImage); // Cache the AHB backed image. Acquires the image here. s->pushImage(image, newTexture); - newImage = true; } texture = newTexture; } // Note that we capture the - mStreamedImageManager.onStreamAcquireImage(texture, stream, newImage); + mStreamedImageManager.onStreamAcquireImage(texture, stream); }); } mStreamsWithPendingAcquiredImage.clear(); @@ -1428,10 +1412,23 @@ void VulkanDriver::updateBufferObject(Handle boh, BufferDescript if (UTILS_UNLIKELY(!mStreamUniformDescriptors.empty())) { auto streamDescriptors = mStreamUniformDescriptors.find(bo.get()); if (streamDescriptors != mStreamUniformDescriptors.end()) { - for (auto const& [offset, stream, associationType]: + for (auto const& [offset, streamHandle, associationType]: streamDescriptors->second.mStreams) { if (associationType == BufferObjectStreamAssociationType::TRANSFORM_MATRIX) { - auto transform = getStreamTransformMatrix(stream); + math::mat3f transform; + if (streamHandle) { + auto stream = + resource_ptr::cast(&mResourceManager, streamHandle); + if (stream->streamType == StreamType::NATIVE) { + FILAMENT_CHECK_PRECONDITION(false) + << "Native Stream not supported in Vulkan."; + } else { + // Backend call since getStreamTransformMatrix is called from async + // updateBufferObject. + transform = stream->getBackEndTransform(); + } + } + copyMat3f(bd.buffer, offset, transform); } } diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index 18ecd0264258..3a1f759e6cdb 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -261,7 +261,18 @@ void VulkanExternalImageManager::bindExternallySampledTexture( fvkmemory::resource_ptr image, SamplerParams samplerParams) { // Should we do duplicate validation here? auto& imageData = findImage(mImages, image); - mSetBindings.push_back({ bindingPoint, imageData.image, set, samplerParams }); + auto itr = std::find_if(mSetBindings.begin(), mSetBindings.end(), + [&](SetBindingInfo const& binding) { + SamplerParams::EqualTo comparator; + return (binding.set == set && binding.binding == bindingPoint && + comparator(samplerParams, binding.samplerParams)); + }); + if (itr == mSetBindings.end()) { + mSetBindings.push_back({ bindingPoint, imageData.image, set, samplerParams }); + } else { + // override the image data in the binding point + itr->image = image; + } } void VulkanExternalImageManager::addExternallySampledTexture( diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp index 7cf3a0b745a3..7ead7b358563 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp @@ -54,17 +54,15 @@ void VulkanStreamedImageManager::unbindStreamedTexture( } void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptr image, - fvkmemory::resource_ptr stream, bool newImage) { + fvkmemory::resource_ptr stream) { for (StreamedTextureBinding const& data: mStreamedTexturesBindings) { // Find the right stream if (data.image->getStream() == stream) { // For some reason, some of the frames coming to us, are on streams where the // descriptor set isn't external... if (data.set->getExternalSamplerVkSet()) { - if (newImage) { - mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, - image, data.samplerParams); - } + mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image, + data.samplerParams); } else { //... In this case we just default to using the normal path and update the // sampler. diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.h b/filament/backend/src/vulkan/VulkanStreamedImageManager.h index 365934eb2bc7..858158cb3893 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.h +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.h @@ -46,7 +46,7 @@ class VulkanStreamedImageManager { void unbindStreamedTexture(fvkmemory::resource_ptr set, uint8_t bindingPoint); void onStreamAcquireImage(fvkmemory::resource_ptr image, - fvkmemory::resource_ptr stream, bool newImage); + fvkmemory::resource_ptr stream); private: struct StreamedTextureBinding { From 86f0afe3ecb7a0af59451b21b7ebb05f160b3720 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Fri, 19 Sep 2025 11:30:47 -0700 Subject: [PATCH 36/43] feedback --- filament/backend/src/vulkan/VulkanDriver.cpp | 2 +- .../backend/src/vulkan/VulkanExternalImageManager.cpp | 5 ++--- filament/backend/src/vulkan/VulkanTexture.h | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 83433b66463b..dda208e32ea1 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1104,6 +1104,7 @@ Handle VulkanDriver::createStreamNative(void* nativeStream, utils::CSt } Handle VulkanDriver::createStreamAcquired(utils::CString tag) { + // @TODO This is still not thread-safe. We might have to revisit this question. FVK_SYSTRACE_SCOPE(); auto handle = mResourceManager.allocHandle(); auto stream = resource_ptr::make(&mResourceManager, handle); @@ -1193,7 +1194,6 @@ void VulkanDriver::updateStreams(CommandStream* driver) { texture = newTexture; } - // Note that we capture the mStreamedImageManager.onStreamAcquireImage(texture, stream); }); } diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index 3a1f759e6cdb..7b6d03d9da0e 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -263,15 +263,14 @@ void VulkanExternalImageManager::bindExternallySampledTexture( auto& imageData = findImage(mImages, image); auto itr = std::find_if(mSetBindings.begin(), mSetBindings.end(), [&](SetBindingInfo const& binding) { - SamplerParams::EqualTo comparator; - return (binding.set == set && binding.binding == bindingPoint && - comparator(samplerParams, binding.samplerParams)); + return (binding.set == set && binding.binding == bindingPoint); }); if (itr == mSetBindings.end()) { mSetBindings.push_back({ bindingPoint, imageData.image, set, samplerParams }); } else { // override the image data in the binding point itr->image = image; + itr->samplerParams = samplerParams; } } diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index 1991013ff644..688e1c45b713 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -59,10 +59,10 @@ struct VulkanStream : public HwStream, fvkmemory::Resource { void setBackendTransform(const math::mat3f& transform) { mTransform = transform; } const math::mat3f& getBackEndTransform() const { return mTransform; } fvkmemory::resource_ptr getTexture(void* ahb) { - fvkmemory::resource_ptr frame; - const auto& it = mTextures.find(ahb); - if (it != mTextures.end()) frame = it->second; - return frame; + if (auto itr = mTextures.find(ahb); itr != mTextures.end()) { + return itr->second; + } + return {}; } void pushImage(void* ahb, fvkmemory::resource_ptr tex) { mTextures[ahb] = tex; } From 44b506b641423fc4921112355cf5b805031071fe Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Mon, 22 Sep 2025 08:52:14 -0700 Subject: [PATCH 37/43] Feedback. --- filament/backend/src/vulkan/VulkanDriver.cpp | 50 ++++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index dda208e32ea1..71fef64dc450 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -146,6 +146,31 @@ static CallbackHandler::Callback syncCallbackWrapper = [](void* userData) { cbData->cb(cbData->sync, cbData->userData); }; +// This needs to be discussed because Vulkan has different default Matrix formats +// right now this is following GL. +static void copyMat3f(void* addr, size_t const offset, const math::mat3f& v) noexcept { + struct mat43 { + float v[3][4]; + }; + + addr = static_cast(addr) + offset; + mat43& temp = *static_cast(addr); + + temp.v[0][0] = v[0][0]; + temp.v[0][1] = v[0][1]; + temp.v[0][2] = v[0][2]; + + temp.v[1][0] = v[1][0]; + temp.v[1][1] = v[1][1]; + temp.v[1][2] = v[1][2]; + + temp.v[2][0] = v[2][0]; + temp.v[2][1] = v[2][1]; + temp.v[2][2] = v[2][2]; + + // don't store anything in temp.v[][3] because there could be uniforms packed there +} + }// anonymous namespace #if FVK_ENABLED(FVK_DEBUG_DEBUG_UTILS) @@ -1476,31 +1501,6 @@ void VulkanDriver::registerBufferObjectStreams(Handle boh, mStreamUniformDescriptors[bo.get()] = std::move(streams); } -// This needs to be discussed because Vulkan has different default Matrix formats -// right now this is following GL. -static void copyMat3f(void* addr, size_t const offset, const math::mat3f& v) noexcept { - struct mat43 { - float v[3][4]; - }; - - addr = static_cast(addr) + offset; - mat43& temp = *static_cast(addr); - - temp.v[0][0] = v[0][0]; - temp.v[0][1] = v[0][1]; - temp.v[0][2] = v[0][2]; - - temp.v[1][0] = v[1][0]; - temp.v[1][1] = v[1][1]; - temp.v[1][2] = v[1][2]; - - temp.v[2][0] = v[2][0]; - temp.v[2][1] = v[2][1]; - temp.v[2][2] = v[2][2]; - - // don't store anything in temp.v[][3] because there could be uniforms packed there -} - void VulkanDriver::updateBufferObject(Handle boh, BufferDescriptor&& bd, uint32_t byteOffset) { VulkanCommandBuffer& commands = mCommands.get(); From df284fe6c82f3baa56946710f80994273a796b27 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Mon, 22 Sep 2025 09:25:03 -0700 Subject: [PATCH 38/43] Nit --- filament/backend/src/vulkan/VulkanDriver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 71fef64dc450..7587983c4d00 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1170,8 +1170,8 @@ void VulkanDriver::updateStreams(CommandStream* driver) { // a copy of a resource_ptr. We only need it find the associated stream // inside the mStreamedImageManager texture bindings driver->queueCommand([this, stream, s = stream.get(), - image = stream->getAcquired().image, - transform = stream->getFrontEndTransform()]() { + image = stream->getAcquired().image, + transform = stream->getFrontEndTransform()]() { auto texture = s->getTexture(image); s->setBackendTransform(transform); if (!texture) { From f96afefb9f84d34121ab701b74df10a7458751c9 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Tue, 23 Sep 2025 08:46:27 -0700 Subject: [PATCH 39/43] Feedback --- filament/backend/src/vulkan/VulkanDriver.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 7587983c4d00..8ede538513dd 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1141,6 +1141,12 @@ void VulkanDriver::setAcquiredImage(Handle sh, void* image, const math CallbackHandler* handler, StreamCallback cb, void* userData) { FVK_SYSTRACE_SCOPE(); auto stream = resource_ptr::cast(&mResourceManager, sh); + assert_invariant(stream->streamType == StreamType::ACQUIRED); + assert_invariant(image != nullptr); + // This covers the case where setAcquiredImage is called back to back (no updateStreams in between) + if (stream->previousNeedsRelease()) { + scheduleRelease(stream->takePrevious()); + } stream->acquire({ image, cb, userData, handler}); stream->setFrontEndTransform(transform); mStreamsWithPendingAcquiredImage.push_back(stream); From e4f29b065e8c20e0185389040d68a4844a41b25d Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 24 Sep 2025 09:13:10 -0700 Subject: [PATCH 40/43] Feedback --- filament/backend/src/vulkan/VulkanStreamedImageManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp index 7ead7b358563..4735b19651be 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp @@ -60,7 +60,7 @@ void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptrgetStream() == stream) { // For some reason, some of the frames coming to us, are on streams where the // descriptor set isn't external... - if (data.set->getExternalSamplerVkSet()) { + if (data.image->getVkFormat() == VK_FORMAT_UNDEFINED) { mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image, data.samplerParams); } else { From 56831e298e402fad4979b56cf4608f83c7ed794b Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 24 Sep 2025 20:09:36 -0700 Subject: [PATCH 41/43] Feedback --- filament/backend/src/vulkan/VulkanDriver.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 8ede538513dd..9e8684c18a99 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -1513,7 +1513,6 @@ void VulkanDriver::updateBufferObject(Handle boh, BufferDescript auto bo = resource_ptr::cast(&mResourceManager, boh); commands.acquire(bo); - bo->loadFromCpu(commands, bd.buffer, byteOffset, bd.size); if (UTILS_UNLIKELY(!mStreamUniformDescriptors.empty())) { auto streamDescriptors = mStreamUniformDescriptors.find(bo.get()); @@ -1542,6 +1541,8 @@ void VulkanDriver::updateBufferObject(Handle boh, BufferDescript } } + bo->loadFromCpu(commands, bd.buffer, byteOffset, bd.size); + scheduleDestroy(std::move(bd)); } From 77b991367ab90067b4ca9e0b1403580ea73e6173 Mon Sep 17 00:00:00 2001 From: Powei Feng Date: Fri, 26 Sep 2025 11:10:57 -0700 Subject: [PATCH 42/43] vk: fix update after bind (wrt descriptor set) validation error In filament, once a descriptor set is bound, we no longer make updates to it. However, this guarrantee was broken in the external sampler/image because a colorspace change might necessitate a layout change, and thereby a new descriptor needs to be generated with the appropriate externally sampled image updated in the new set. Another use case is the Stream API, where each frame we might be getting a different AHardwareBuffer, meaning we need to update one or multiple sampler bindings in a set that might have been bound in a previous frame. In this change, we always create a new set when there's a change in the image currently bound to an existing set (while accounting for whether we need to use a new externally sampled layout or not). --- .../src/vulkan/VulkanDescriptorSetCache.cpp | 87 ++++++++++++++----- .../src/vulkan/VulkanDescriptorSetCache.h | 7 +- .../src/vulkan/VulkanExternalImageManager.cpp | 59 +------------ filament/backend/src/vulkan/VulkanHandles.cpp | 64 +++++++++++++- filament/backend/src/vulkan/VulkanHandles.h | 61 ++++++------- 5 files changed, 160 insertions(+), 118 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDescriptorSetCache.cpp b/filament/backend/src/vulkan/VulkanDescriptorSetCache.cpp index 3fc5697c5ad3..1ebc49584c29 100644 --- a/filament/backend/src/vulkan/VulkanDescriptorSetCache.cpp +++ b/filament/backend/src/vulkan/VulkanDescriptorSetCache.cpp @@ -19,6 +19,7 @@ #include "VulkanCommands.h" #include "VulkanHandles.h" #include "VulkanConstants.h" +#include "utils/compiler.h" #include #include @@ -31,6 +32,18 @@ namespace filament::backend { namespace { +using Bitmask = fvkutils::UniformBufferBitmask; +static_assert(sizeof(Bitmask) * 8 == fvkutils::MAX_DESCRIPTOR_SET_BITMASK_BITS); + +Bitmask foldBitsInHalf(Bitmask bitset) { + Bitmask outBitset; + bitset.forEachSetBit([&](size_t index) { + constexpr size_t BITMASK_LOWER_BITS_LEN = sizeof(outBitset) * 4; + outBitset.set(index % BITMASK_LOWER_BITS_LEN); + }); + return outBitset; +} + using DescriptorSetLayoutArray = VulkanDescriptorSetCache::DescriptorSetLayoutArray; using DescriptorCount = VulkanDescriptorSetCache::DescriptorCount; @@ -304,8 +317,7 @@ void VulkanDescriptorSetCache::commit(VulkanCommandBuffer* commands, // This code actually binds the descriptor sets. auto set = updateSets[index]; VkCommandBuffer const cmdbuffer = commands->buffer(); - VkDescriptorSet vkset = useExternalSamplers[index] ? set->getExternalSamplerVkSet() : - set->getVkSet(); + VkDescriptorSet const vkset = set->getVkSet(); commands->acquire(set); vkCmdBindDescriptorSets(cmdbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, index, 1, &vkset, set->uniqueDynamicUboCount, set->getOffsets()->data()); @@ -341,17 +353,39 @@ void VulkanDescriptorSetCache::updateBuffer(fvkmemory::resource_ptrgetExternalSamplerVkSet(); - externalSamplerSet != VK_NULL_HANDLE) { - descriptorWrite.dstSet = externalSamplerSet; - vkUpdateDescriptorSets(mDevice, 1, &descriptorWrite, 0, nullptr); - } set->acquire(bufferObject); } -void VulkanDescriptorSetCache::updateSamplerImpl(VkDescriptorSet vkset, uint8_t binding, - fvkmemory::resource_ptr texture, VkSampler sampler) noexcept { +void VulkanDescriptorSetCache::updateSampler(fvkmemory::resource_ptr set, + uint8_t binding, fvkmemory::resource_ptr texture, + VkSampler sampler, VkDescriptorSetLayout externalSamplerLayout) noexcept { + + // We have to update a bound set for two use cases + // - streaming API (a changing feed of AHardwareBuffer) + // - external samplers - potential changing of dataspace per-frame + if (UTILS_UNLIKELY(set->isBound())) { + auto layout = set->getLayout(); + // Build a new descriptor set from the new layout + auto genLayout = externalSamplerLayout != VK_NULL_HANDLE ? + externalSamplerLayout : layout->getVkLayout(); + VkDescriptorSet const newSet = getVkSet(layout->count, genLayout); + Bitmask const ubo = layout->bitmask.ubo | layout->bitmask.dynamicUbo; + Bitmask samplers = layout->bitmask.sampler; + samplers.unset(binding); + + // Each bitmask denotes a binding index, and separated into two stages - vertex and buffer + // We fold the two stages into just the lower half of the bits to denote a combined set of + // bindings. + Bitmask const copyBindings = foldBitsInHalf(ubo | samplers); + VkDescriptorSet const srcSet = set->getVkSet(); + copySet(srcSet, newSet, copyBindings); + set->addNewSet(newSet, + [this, layoutCount = layout->count, genLayout, newSet](VulkanDescriptorSet*) { + this->manualRecycle(layoutCount, genLayout, newSet); + }); + } + + VkDescriptorSet const vkset = set->getVkSet(); VkImageSubresourceRange range = texture->getPrimaryViewRange(); VkImageViewType const expectedType = texture->getViewType(); if (any(texture->usage & TextureUsage::DEPTH_ATTACHMENT) && @@ -376,19 +410,6 @@ void VulkanDescriptorSetCache::updateSamplerImpl(VkDescriptorSet vkset, uint8_t .pImageInfo = &info, }; vkUpdateDescriptorSets(mDevice, 1, &descriptorWrite, 0, nullptr); -} - -void VulkanDescriptorSetCache::updateSampler(fvkmemory::resource_ptr set, - uint8_t binding, fvkmemory::resource_ptr texture, - VkSampler sampler) noexcept { - updateSamplerImpl(set->getVkSet(), binding, texture, sampler); - set->acquire(texture); -} - -void VulkanDescriptorSetCache::updateSamplerForExternalSamplerSet( - fvkmemory::resource_ptr set, uint8_t binding, - fvkmemory::resource_ptr texture) noexcept { - updateSamplerImpl(set->getExternalSamplerVkSet(), binding, texture, VK_NULL_HANDLE); set->acquire(texture); } @@ -405,7 +426,7 @@ fvkmemory::resource_ptr VulkanDescriptorSetCache::createSet auto const& count = layout->count; auto const vklayout = layout->getVkLayout(); auto set = fvkmemory::resource_ptr::make( - mResourceManager, handle, layout->bitmask.dynamicUbo, layout->count.dynamicUbo, + mResourceManager, handle, layout, [vkSet, count, vklayout, this]( VulkanDescriptorSet*) { this->manualRecycle(count, vklayout, vkSet); }, vkSet); @@ -428,4 +449,22 @@ void VulkanDescriptorSetCache::manualRecycle(VulkanDescriptorSetLayout::Count co void VulkanDescriptorSetCache::gc() { mStashedSets = {}; } +void VulkanDescriptorSetCache::copySet(VkDescriptorSet srcSet, VkDescriptorSet dstSet, + fvkutils::SamplerBitmask bindings) const { + // TODO: fix the size for better memory management + std::vector copies; + bindings.forEachSetBit([&](size_t index) { + copies.push_back({ + .sType = VK_STRUCTURE_TYPE_COPY_DESCRIPTOR_SET, + .srcSet = srcSet, + .srcBinding = (uint32_t) index, + .dstSet = dstSet, + .dstBinding = (uint32_t) index, + .descriptorCount = 1, + }); + }); + vkUpdateDescriptorSets(mDevice, 0, nullptr, copies.size(), copies.data()); +} + + } // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanDescriptorSetCache.h b/filament/backend/src/vulkan/VulkanDescriptorSetCache.h index d19209fbcdcc..135fdad59e30 100644 --- a/filament/backend/src/vulkan/VulkanDescriptorSetCache.h +++ b/filament/backend/src/vulkan/VulkanDescriptorSetCache.h @@ -55,7 +55,8 @@ class VulkanDescriptorSetCache { VkDeviceSize size) noexcept; void updateSampler(fvkmemory::resource_ptr set, uint8_t binding, - fvkmemory::resource_ptr texture, VkSampler sampler) noexcept; + fvkmemory::resource_ptr texture, VkSampler sampler, + VkDescriptorSetLayout externalSamplerLayout = VK_NULL_HANDLE) noexcept; void updateSamplerForExternalSamplerSet(fvkmemory::resource_ptr set, uint8_t binding, fvkmemory::resource_ptr texture) noexcept; @@ -90,8 +91,8 @@ class VulkanDescriptorSetCache { void resetCachedState() noexcept { mLastBoundInfo = {}; } private: - void updateSamplerImpl(VkDescriptorSet set, uint8_t binding, - fvkmemory::resource_ptr texture, VkSampler sampler) noexcept; + void copySet(VkDescriptorSet srcSet, VkDescriptorSet destSet, + fvkutils::SamplerBitmask copyBindings) const; class DescriptorInfinitePool; diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index 18ecd0264258..79f4da99c30f 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -31,9 +31,6 @@ namespace filament::backend { namespace { -using Bitmask = fvkutils::UniformBufferBitmask; -static_assert(sizeof(Bitmask) * 8 == fvkutils::MAX_DESCRIPTOR_SET_BITMASK_BITS); - template void erasep(std::vector& v, std::function f) { auto newEnd = std::remove_if(v.begin(), v.end(), f); @@ -50,31 +47,6 @@ ImageData& findImage(std::vector& images, return *itr; } -void copySet(VkDevice device, VkDescriptorSet srcSet, VkDescriptorSet dstSet, Bitmask bindings) { - // TODO: fix the size for better memory management - std::vector copies; - bindings.forEachSetBit([&](size_t index) { - copies.push_back({ - .sType = VK_STRUCTURE_TYPE_COPY_DESCRIPTOR_SET, - .srcSet = srcSet, - .srcBinding = (uint32_t) index, - .dstSet = dstSet, - .dstBinding = (uint32_t) index, - .descriptorCount = 1, - }); - }); - vkUpdateDescriptorSets(device, 0, nullptr, copies.size(), copies.data()); -} - -Bitmask foldBitsInHalf(Bitmask bitset) { - Bitmask outBitset; - bitset.forEachSetBit([&](size_t index) { - constexpr size_t BITMASK_LOWER_BITS_LEN = sizeof(outBitset) * 4; - outBitset.set(index % BITMASK_LOWER_BITS_LEN); - }); - return outBitset; -} - }// namespace VulkanExternalImageManager::VulkanExternalImageManager(VulkanPlatform* platform, @@ -172,38 +144,13 @@ void VulkanExternalImageManager::updateSetAndLayout( std::for_each(samplerAndBindings.begin(), samplerAndBindings.end(), [&](auto const& b) { outSamplers.push_back(std::get<1>(b)); }); - VkDescriptorSetLayout const oldLayout = layout->getExternalSamplerVkLayout(); VkDescriptorSetLayout const newLayout = mDescriptorSetLayoutCache->getVkLayout(layout->bitmask, actualExternalSamplers, outSamplers); - - // Need to copy the set - VkDescriptorSet const oldSet = set->getExternalSamplerVkSet(); - if (oldLayout != newLayout || oldSet == VK_NULL_HANDLE) { - // Build a new descriptor set from the new layout - VkDescriptorSet const newSet = mDescriptorSetCache->getVkSet(layout->count, newLayout); - auto const ubo = layout->bitmask.ubo | layout->bitmask.dynamicUbo; - auto const samplers = layout->bitmask.sampler & (~actualExternalSamplers); - - // Each bitmask denotes a binding index, and separated into two stages - vertex and buffer - // We fold the two stages into just the lower half of the bits to denote a combined set of - // bindings. - Bitmask const copyBindings = foldBitsInHalf(ubo | samplers); - VkDescriptorSet const srcSet = oldSet != VK_NULL_HANDLE ? oldSet : set->getVkSet(); - copySet(mPlatform->getDevice(), srcSet, newSet, copyBindings); - - set->setExternalSamplerVkSet(newSet, - [&descriptorSetCache = mDescriptorSetCache, layoutCount = layout->count, newLayout, - newSet](VulkanDescriptorSet*) { - descriptorSetCache->manualRecycle(layoutCount, newLayout, newSet); - }); - if (oldLayout != newLayout) { - layout->setExternalSamplerVkLayout(newLayout); - } - } - + layout->setExternalSamplerVkLayout(newLayout); // Update the external samplers in the set for (auto& [binding, sampler, image]: samplerAndBindings) { - mDescriptorSetCache->updateSamplerForExternalSamplerSet(set, binding, image); + mDescriptorSetCache->updateSampler(set, binding, image, VK_NULL_HANDLE /*sampler*/, + newLayout); } } diff --git a/filament/backend/src/vulkan/VulkanHandles.cpp b/filament/backend/src/vulkan/VulkanHandles.cpp index c2eef217a48b..5b72ae402020 100644 --- a/filament/backend/src/vulkan/VulkanHandles.cpp +++ b/filament/backend/src/vulkan/VulkanHandles.cpp @@ -40,6 +40,11 @@ namespace filament::backend { namespace { +// We don't have a good estimate for this magic number. This is used to remove empty +// slots from a descriptor set (a VulkanDescriptorset is backed by multiple +// VkDescriptorSet. +size_t DESCRIPTOR_SET_GC_LIMIT = 10; + inline VulkanBufferBinding getBufferObjectBinding(BufferObjectBinding bindingType) noexcept { switch (bindingType) { case BufferObjectBinding::VERTEX: @@ -192,15 +197,70 @@ VulkanDescriptorSetLayout::Bitmask VulkanDescriptorSetLayout::Bitmask::fromLayou return fromBackendLayout(layout); } -// This method will store an age associated with this command buffer into the VulkanBuffer, which -// will allow us to determine whether a barrier is necessary or not. +VulkanDescriptorSet::~VulkanDescriptorSet() { + for (auto setBundle: mSets) { + if (setBundle.onRecycleFn) { + setBundle.onRecycleFn(this); + } + } +} + +VulkanDescriptorSet::VulkanDescriptorSet(fvkmemory::resource_ptr layout, + OnRecycle&& onRecycleFn, VkDescriptorSet vkSet) + : dynamicUboMask(layout->bitmask.dynamicUbo), + uniqueDynamicUboCount(layout->count.dynamicUbo), + mLayout(layout), + mCurrentSetIndex(0) { + addNewSet(vkSet, std::move(onRecycleFn)); +} + void VulkanDescriptorSet::referencedBy(VulkanCommandBuffer& commands) { + // This will store an age associated with this command buffer into the VulkanBuffer, which + // will allow us to determine whether a barrier is necessary or not. mUboMask.forEachSetBit([this, &commands](size_t index) { auto& res = mResources[index]; fvkmemory::resource_ptr bo = fvkmemory::resource_ptr::cast((VulkanBufferObject*) res.get()); bo->referencedBy(commands); }); + mSets[mCurrentSetIndex].fenceStatus = commands.getFenceStatus(); +} + +void VulkanDescriptorSet::gc() { + size_t empty = 0; + for (auto& setBundle: mSets) { + assert_invariant(setBundle.onRecycleFn); + if (setBundle.vkSet == VK_NULL_HANDLE) { + empty++; + continue; + } + if (setBundle.fenceStatus && setBundle.fenceStatus->getStatus() == VK_SUCCESS) { + setBundle.onRecycleFn(this); + setBundle.vkSet = VK_NULL_HANDLE; + setBundle.fenceStatus = {}; + empty++; + } + } + + if (empty > DESCRIPTOR_SET_GC_LIMIT) { + std::vector retainedSets; + for (auto& setBundle: mSets) { + if (setBundle.vkSet != VK_NULL_HANDLE) { + retainedSets.push_back({ + setBundle.vkSet, + std::move(setBundle.onRecycleFn), + std::move(setBundle.fenceStatus) + }); + } + } + std::swap(mSets, retainedSets); + } +} + +void VulkanDescriptorSet::addNewSet(VkDescriptorSet vkSet, OnRecycle&& onRecycleFn) { + gc(); + mCurrentSetIndex = mSets.size(); + mSets.push_back({ vkSet, std::move(onRecycleFn) }); } PushConstantDescription::PushConstantDescription(backend::Program const& program) { diff --git a/filament/backend/src/vulkan/VulkanHandles.h b/filament/backend/src/vulkan/VulkanHandles.h index e185d750be26..50277639f14d 100644 --- a/filament/backend/src/vulkan/VulkanHandles.h +++ b/filament/backend/src/vulkan/VulkanHandles.h @@ -28,6 +28,7 @@ #include "VulkanTexture.h" #include "vulkan/VulkanCommands.h" #include "vulkan/memory/Resource.h" +#include "vulkan/memory/ResourcePointer.h" #include "vulkan/utils/Definitions.h" #include "vulkan/utils/StaticVector.h" @@ -57,6 +58,8 @@ inline uint8_t collapsedCount(Bitmask const& mask) { } // anonymous namespace +class VulkanDescriptorSetCache; + struct VulkanBufferObject; struct VulkanDescriptorSetLayout : public HwDescriptorSetLayout, fvkmemory::Resource { @@ -159,38 +162,14 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { using OnRecycle = std::function; VulkanDescriptorSet( - fvkutils::UniformBufferBitmask const& dynamicUboMask, - uint8_t uniqueDynamicUboCount, - OnRecycle&& onRecycleFn, VkDescriptorSet vkSet) - : dynamicUboMask(dynamicUboMask), - uniqueDynamicUboCount(uniqueDynamicUboCount), - mVkSet(vkSet), - mOnRecycleFn(std::move(onRecycleFn)) {} + fvkmemory::resource_ptr layout, + OnRecycle&& onRecycleFn, VkDescriptorSet vkSet); // NOLINTNEXTLINE(bugprone-exception-escape) - ~VulkanDescriptorSet() { - if (mOnRecycleFn) { - mOnRecycleFn(this); - } - if (mOnRecycleExternalSamplerFn) { - mOnRecycleExternalSamplerFn(this); - } - } + ~VulkanDescriptorSet(); VkDescriptorSet getVkSet() const noexcept { - return mVkSet; - } - - VkDescriptorSet getExternalSamplerVkSet() const noexcept { - return mExternalSamplerVkSet; - } - - void setExternalSamplerVkSet(VkDescriptorSet vkset, OnRecycle onRecycle) { - mExternalSamplerVkSet = vkset; - if (mOnRecycleExternalSamplerFn) { - mOnRecycleExternalSamplerFn(this); - } - mOnRecycleExternalSamplerFn = onRecycle; + return mSets[mCurrentSetIndex].vkSet; } void setOffsets(backend::DescriptorSetOffsetArray&& offsets) noexcept { @@ -211,17 +190,33 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { void referencedBy(VulkanCommandBuffer& commands); - fvkutils::UniformBufferBitmask const dynamicUboMask; + bool isBound() const { + return bool(mSets[mCurrentSetIndex].fenceStatus); + } + + fvkmemory::resource_ptr getLayout() const { return mLayout; } + + fvkutils::UniformBufferBitmask const& dynamicUboMask; uint8_t const uniqueDynamicUboCount; private: - VkDescriptorSet const mVkSet; - VkDescriptorSet mExternalSamplerVkSet = VK_NULL_HANDLE; + friend class VulkanDescriptorSetCache; + + void addNewSet(VkDescriptorSet vkSet, OnRecycle&& onRecycleFn); + + void gc(); + + struct InternalVkSet { + VkDescriptorSet vkSet = {}; + OnRecycle onRecycleFn; + std::shared_ptr fenceStatus; + }; + fvkmemory::resource_ptr mLayout; backend::DescriptorSetOffsetArray mOffsets; std::vector> mResources; - OnRecycle mOnRecycleFn; - OnRecycle mOnRecycleExternalSamplerFn; + uint8_t mCurrentSetIndex; + std::vector mSets; fvkutils::UniformBufferBitmask mUboMask; }; From ffd6e28af07b6f369a492b4b7a1e49c1409ad6b6 Mon Sep 17 00:00:00 2001 From: Serge Metral Date: Wed, 29 Oct 2025 15:50:05 +0000 Subject: [PATCH 43/43] Removing the decision on whether to use external format or not out of the streamed texture manager. The issue is that all textures are external, not all are using external format. But it's more robust to let the external format image manager rely on the AHB* to decide if the format is external or not. Regardless (because the memory is external) we need to send this to the external image manager. --- filament/backend/src/vulkan/VulkanDriver.cpp | 2 +- .../src/vulkan/VulkanStreamedImageManager.cpp | 27 +++---------------- .../src/vulkan/VulkanStreamedImageManager.h | 6 +---- 3 files changed, 6 insertions(+), 29 deletions(-) diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 0ced39fae7b6..3cb3500d89cf 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -269,7 +269,7 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext& context, mQueryManager(mPlatform->getDevice()), mExternalImageManager(platform, &mSamplerCache, &mYcbcrConversionCache, &mDescriptorSetCache, &mDescriptorSetLayoutCache), - mStreamedImageManager(&mExternalImageManager, &mDescriptorSetCache, &mSamplerCache), + mStreamedImageManager(&mExternalImageManager), mIsSRGBSwapChainSupported(mPlatform->getCustomization().isSRGBSwapChainSupported), mIsMSAASwapChainSupported(false), // TODO: support MSAA swapchain mStereoscopicType(driverConfig.stereoscopicType) { diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp index 4735b19651be..cf5a2ac543b9 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp @@ -22,15 +22,8 @@ #include "VulkanSamplerCache.h" namespace filament::backend { - VulkanStreamedImageManager::VulkanStreamedImageManager( - VulkanExternalImageManager* manager, - VulkanDescriptorSetCache* descriptorSet, - VulkanSamplerCache* samplerCache) - : mExternalImageManager(manager), - mDescriptorSetCache(descriptorSet), - mSamplerCache(samplerCache) { - -} +VulkanStreamedImageManager::VulkanStreamedImageManager(VulkanExternalImageManager* manager) + : mExternalImageManager(manager) {} VulkanStreamedImageManager::~VulkanStreamedImageManager() = default; @@ -58,20 +51,8 @@ void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptrgetStream() == stream) { - // For some reason, some of the frames coming to us, are on streams where the - // descriptor set isn't external... - if (data.image->getVkFormat() == VK_FORMAT_UNDEFINED) { - mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image, - data.samplerParams); - } else { - //... In this case we just default to using the normal path and update the - // sampler. - VulkanSamplerCache::Params cacheParams = { - .sampler = data.samplerParams, - }; - VkSampler const vksampler = mSamplerCache->getSampler(cacheParams); - mDescriptorSetCache->updateSampler(data.set, data.binding, image, vksampler); - } + mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image, + data.samplerParams); } } } diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.h b/filament/backend/src/vulkan/VulkanStreamedImageManager.h index 858158cb3893..718e9693203a 100644 --- a/filament/backend/src/vulkan/VulkanStreamedImageManager.h +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.h @@ -34,9 +34,7 @@ class VulkanSamplerCache; class VulkanStreamedImageManager { public: VulkanStreamedImageManager( - VulkanExternalImageManager* manager, - VulkanDescriptorSetCache* descriptorSet, - VulkanSamplerCache* samplerCache); + VulkanExternalImageManager* manager); ~VulkanStreamedImageManager(); void terminate(); @@ -59,8 +57,6 @@ class VulkanStreamedImageManager { std::vector mStreamedTexturesBindings; VulkanExternalImageManager* mExternalImageManager; - VulkanDescriptorSetCache* mDescriptorSetCache; - VulkanSamplerCache* mSamplerCache; }; } // namespace filament::backend