diff --git a/filament/backend/CMakeLists.txt b/filament/backend/CMakeLists.txt index 058536689f6a..c63ef9efa5b6 100644 --- a/filament/backend/CMakeLists.txt +++ b/filament/backend/CMakeLists.txt @@ -204,6 +204,8 @@ if (FILAMENT_SUPPORTS_VULKAN) src/vulkan/VulkanDriverFactory.h src/vulkan/VulkanExternalImageManager.cpp src/vulkan/VulkanExternalImageManager.h + src/vulkan/VulkanStreamedImageManager.cpp + src/vulkan/VulkanStreamedImageManager.h src/vulkan/VulkanFboCache.cpp src/vulkan/VulkanFboCache.h src/vulkan/VulkanHandles.cpp diff --git a/filament/backend/include/backend/Platform.h b/filament/backend/include/backend/Platform.h index 2b2fc1678f6b..0202bb232bc7 100644 --- a/filament/backend/include/backend/Platform.h +++ b/filament/backend/include/backend/Platform.h @@ -70,6 +70,9 @@ class UTILS_PUBLIC Platform { ExternalImageHandle& operator=(ExternalImageHandle const& rhs) noexcept; ExternalImageHandle& operator=(ExternalImageHandle&& rhs) noexcept; + bool operator==(const ExternalImageHandle& rhs) const noexcept { + return mTarget == rhs.mTarget; + } explicit operator bool() const noexcept { return mTarget != nullptr; } ExternalImage* UTILS_NULLABLE get() noexcept { return mTarget; } diff --git a/filament/backend/src/vulkan/VulkanDescriptorSetCache.cpp b/filament/backend/src/vulkan/VulkanDescriptorSetCache.cpp index 3fc5697c5ad3..1ebc49584c29 100644 --- a/filament/backend/src/vulkan/VulkanDescriptorSetCache.cpp +++ b/filament/backend/src/vulkan/VulkanDescriptorSetCache.cpp @@ -19,6 +19,7 @@ #include "VulkanCommands.h" #include "VulkanHandles.h" #include "VulkanConstants.h" +#include "utils/compiler.h" #include #include @@ -31,6 +32,18 @@ namespace filament::backend { namespace { +using Bitmask = fvkutils::UniformBufferBitmask; +static_assert(sizeof(Bitmask) * 8 == fvkutils::MAX_DESCRIPTOR_SET_BITMASK_BITS); + +Bitmask foldBitsInHalf(Bitmask bitset) { + Bitmask outBitset; + bitset.forEachSetBit([&](size_t index) { + constexpr size_t BITMASK_LOWER_BITS_LEN = sizeof(outBitset) * 4; + outBitset.set(index % BITMASK_LOWER_BITS_LEN); + }); + return outBitset; +} + using DescriptorSetLayoutArray = VulkanDescriptorSetCache::DescriptorSetLayoutArray; using DescriptorCount = VulkanDescriptorSetCache::DescriptorCount; @@ -304,8 +317,7 @@ void VulkanDescriptorSetCache::commit(VulkanCommandBuffer* commands, // This code actually binds the descriptor sets. auto set = updateSets[index]; VkCommandBuffer const cmdbuffer = commands->buffer(); - VkDescriptorSet vkset = useExternalSamplers[index] ? set->getExternalSamplerVkSet() : - set->getVkSet(); + VkDescriptorSet const vkset = set->getVkSet(); commands->acquire(set); vkCmdBindDescriptorSets(cmdbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, index, 1, &vkset, set->uniqueDynamicUboCount, set->getOffsets()->data()); @@ -341,17 +353,39 @@ void VulkanDescriptorSetCache::updateBuffer(fvkmemory::resource_ptrgetExternalSamplerVkSet(); - externalSamplerSet != VK_NULL_HANDLE) { - descriptorWrite.dstSet = externalSamplerSet; - vkUpdateDescriptorSets(mDevice, 1, &descriptorWrite, 0, nullptr); - } set->acquire(bufferObject); } -void VulkanDescriptorSetCache::updateSamplerImpl(VkDescriptorSet vkset, uint8_t binding, - fvkmemory::resource_ptr texture, VkSampler sampler) noexcept { +void VulkanDescriptorSetCache::updateSampler(fvkmemory::resource_ptr set, + uint8_t binding, fvkmemory::resource_ptr texture, + VkSampler sampler, VkDescriptorSetLayout externalSamplerLayout) noexcept { + + // We have to update a bound set for two use cases + // - streaming API (a changing feed of AHardwareBuffer) + // - external samplers - potential changing of dataspace per-frame + if (UTILS_UNLIKELY(set->isBound())) { + auto layout = set->getLayout(); + // Build a new descriptor set from the new layout + auto genLayout = externalSamplerLayout != VK_NULL_HANDLE ? + externalSamplerLayout : layout->getVkLayout(); + VkDescriptorSet const newSet = getVkSet(layout->count, genLayout); + Bitmask const ubo = layout->bitmask.ubo | layout->bitmask.dynamicUbo; + Bitmask samplers = layout->bitmask.sampler; + samplers.unset(binding); + + // Each bitmask denotes a binding index, and separated into two stages - vertex and buffer + // We fold the two stages into just the lower half of the bits to denote a combined set of + // bindings. + Bitmask const copyBindings = foldBitsInHalf(ubo | samplers); + VkDescriptorSet const srcSet = set->getVkSet(); + copySet(srcSet, newSet, copyBindings); + set->addNewSet(newSet, + [this, layoutCount = layout->count, genLayout, newSet](VulkanDescriptorSet*) { + this->manualRecycle(layoutCount, genLayout, newSet); + }); + } + + VkDescriptorSet const vkset = set->getVkSet(); VkImageSubresourceRange range = texture->getPrimaryViewRange(); VkImageViewType const expectedType = texture->getViewType(); if (any(texture->usage & TextureUsage::DEPTH_ATTACHMENT) && @@ -376,19 +410,6 @@ void VulkanDescriptorSetCache::updateSamplerImpl(VkDescriptorSet vkset, uint8_t .pImageInfo = &info, }; vkUpdateDescriptorSets(mDevice, 1, &descriptorWrite, 0, nullptr); -} - -void VulkanDescriptorSetCache::updateSampler(fvkmemory::resource_ptr set, - uint8_t binding, fvkmemory::resource_ptr texture, - VkSampler sampler) noexcept { - updateSamplerImpl(set->getVkSet(), binding, texture, sampler); - set->acquire(texture); -} - -void VulkanDescriptorSetCache::updateSamplerForExternalSamplerSet( - fvkmemory::resource_ptr set, uint8_t binding, - fvkmemory::resource_ptr texture) noexcept { - updateSamplerImpl(set->getExternalSamplerVkSet(), binding, texture, VK_NULL_HANDLE); set->acquire(texture); } @@ -405,7 +426,7 @@ fvkmemory::resource_ptr VulkanDescriptorSetCache::createSet auto const& count = layout->count; auto const vklayout = layout->getVkLayout(); auto set = fvkmemory::resource_ptr::make( - mResourceManager, handle, layout->bitmask.dynamicUbo, layout->count.dynamicUbo, + mResourceManager, handle, layout, [vkSet, count, vklayout, this]( VulkanDescriptorSet*) { this->manualRecycle(count, vklayout, vkSet); }, vkSet); @@ -428,4 +449,22 @@ void VulkanDescriptorSetCache::manualRecycle(VulkanDescriptorSetLayout::Count co void VulkanDescriptorSetCache::gc() { mStashedSets = {}; } +void VulkanDescriptorSetCache::copySet(VkDescriptorSet srcSet, VkDescriptorSet dstSet, + fvkutils::SamplerBitmask bindings) const { + // TODO: fix the size for better memory management + std::vector copies; + bindings.forEachSetBit([&](size_t index) { + copies.push_back({ + .sType = VK_STRUCTURE_TYPE_COPY_DESCRIPTOR_SET, + .srcSet = srcSet, + .srcBinding = (uint32_t) index, + .dstSet = dstSet, + .dstBinding = (uint32_t) index, + .descriptorCount = 1, + }); + }); + vkUpdateDescriptorSets(mDevice, 0, nullptr, copies.size(), copies.data()); +} + + } // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanDescriptorSetCache.h b/filament/backend/src/vulkan/VulkanDescriptorSetCache.h index d19209fbcdcc..135fdad59e30 100644 --- a/filament/backend/src/vulkan/VulkanDescriptorSetCache.h +++ b/filament/backend/src/vulkan/VulkanDescriptorSetCache.h @@ -55,7 +55,8 @@ class VulkanDescriptorSetCache { VkDeviceSize size) noexcept; void updateSampler(fvkmemory::resource_ptr set, uint8_t binding, - fvkmemory::resource_ptr texture, VkSampler sampler) noexcept; + fvkmemory::resource_ptr texture, VkSampler sampler, + VkDescriptorSetLayout externalSamplerLayout = VK_NULL_HANDLE) noexcept; void updateSamplerForExternalSamplerSet(fvkmemory::resource_ptr set, uint8_t binding, fvkmemory::resource_ptr texture) noexcept; @@ -90,8 +91,8 @@ class VulkanDescriptorSetCache { void resetCachedState() noexcept { mLastBoundInfo = {}; } private: - void updateSamplerImpl(VkDescriptorSet set, uint8_t binding, - fvkmemory::resource_ptr texture, VkSampler sampler) noexcept; + void copySet(VkDescriptorSet srcSet, VkDescriptorSet destSet, + fvkutils::SamplerBitmask copyBindings) const; class DescriptorInfinitePool; diff --git a/filament/backend/src/vulkan/VulkanDriver.cpp b/filament/backend/src/vulkan/VulkanDriver.cpp index 510af3e3acda..974385b7f14d 100644 --- a/filament/backend/src/vulkan/VulkanDriver.cpp +++ b/filament/backend/src/vulkan/VulkanDriver.cpp @@ -31,6 +31,7 @@ #include "vulkan/memory/ResourcePointer.h" #include "vulkan/utils/Conversion.h" #include "vulkan/utils/Definitions.h" +#include "vulkan/utils/Image.h" #include #include @@ -148,6 +149,31 @@ static CallbackHandler::Callback syncCallbackWrapper = [](void* userData) { cbData->cb(cbData->sync, cbData->userData); }; +// This needs to be discussed because Vulkan has different default Matrix formats +// right now this is following GL. +static void copyMat3f(void* addr, size_t const offset, const math::mat3f& v) noexcept { + struct mat43 { + float v[3][4]; + }; + + addr = static_cast(addr) + offset; + mat43& temp = *static_cast(addr); + + temp.v[0][0] = v[0][0]; + temp.v[0][1] = v[0][1]; + temp.v[0][2] = v[0][2]; + + temp.v[1][0] = v[1][0]; + temp.v[1][1] = v[1][1]; + temp.v[1][2] = v[1][2]; + + temp.v[2][0] = v[2][0]; + temp.v[2][1] = v[2][1]; + temp.v[2][2] = v[2][2]; + + // don't store anything in temp.v[][3] because there could be uniforms packed there +} + }// anonymous namespace #if FVK_ENABLED(FVK_DEBUG_DEBUG_UTILS) @@ -243,6 +269,7 @@ VulkanDriver::VulkanDriver(VulkanPlatform* platform, VulkanContext& context, mQueryManager(mPlatform->getDevice()), mExternalImageManager(platform, &mSamplerCache, &mYcbcrConversionCache, &mDescriptorSetCache, &mDescriptorSetLayoutCache), + mStreamedImageManager(&mExternalImageManager), mIsSRGBSwapChainSupported(mPlatform->getCustomization().isSRGBSwapChainSupported), mIsMSAASwapChainSupported(false), // TODO: support MSAA swapchain mStereoscopicType(driverConfig.stereoscopicType) { @@ -478,6 +505,9 @@ void VulkanDriver::updateDescriptorSetTexture( if (UTILS_UNLIKELY(mExternalImageManager.isExternallySampledTexture(texture))) { mExternalImageManager.bindExternallySampledTexture(set, binding, texture, params); mAppState.hasBoundExternalImages = true; + } else if (bool(texture->getStream())) { + mStreamedImageManager.bindStreamedTexture(set, binding, texture, params); + mAppState.hasBoundExternalImages = true; } else { VulkanSamplerCache::Params cacheParams = { .sampler = params, @@ -485,6 +515,7 @@ void VulkanDriver::updateDescriptorSetTexture( VkSampler const vksampler = mSamplerCache.getSampler(cacheParams); mDescriptorSetCache.updateSampler(set, binding, texture, vksampler); mExternalImageManager.clearTextureBinding(set, binding); + mStreamedImageManager.unbindStreamedTexture(set, binding); } } @@ -1109,6 +1140,12 @@ void VulkanDriver::destroySwapChain(Handle sch) { } void VulkanDriver::destroyStream(Handle sh) { + FVK_SYSTRACE_SCOPE(); + if (!sh) { + return; + } + auto stream = resource_ptr::cast(&mResourceManager, sh); + stream.dec(); } void VulkanDriver::destroySync(Handle sh) { @@ -1146,18 +1183,39 @@ void VulkanDriver::destroyDescriptorSet(Handle dsh) { } Handle VulkanDriver::createStreamNative(void* nativeStream, utils::ImmutableCString tag) { + FILAMENT_CHECK_PRECONDITION(false) << "createStreamNative not supported in Vulkan."; return {}; } Handle VulkanDriver::createStreamAcquired(utils::ImmutableCString tag) { - return {}; + // @TODO This is still not thread-safe. We might have to revisit this question. + FVK_SYSTRACE_SCOPE(); + auto handle = mResourceManager.allocHandle(); + auto stream = resource_ptr::make(&mResourceManager, handle); + stream.inc(); + return handle; } void VulkanDriver::setAcquiredImage(Handle sh, void* image, const math::mat3f& transform, CallbackHandler* handler, StreamCallback cb, void* userData) { + FVK_SYSTRACE_SCOPE(); + auto stream = resource_ptr::cast(&mResourceManager, sh); + assert_invariant(stream->streamType == StreamType::ACQUIRED); + assert_invariant(image != nullptr); + // This covers the case where setAcquiredImage is called back to back (no updateStreams in between) + if (stream->previousNeedsRelease()) { + scheduleRelease(stream->takePrevious()); + } + stream->acquire({ image, cb, userData, handler}); + stream->setFrontEndTransform(transform); + mStreamsWithPendingAcquiredImage.push_back(stream); } void VulkanDriver::setStreamDimensions(Handle sh, uint32_t width, uint32_t height) { + FVK_SYSTRACE_SCOPE(); + auto stream = resource_ptr::cast(&mResourceManager, sh); + stream->width = width; + stream->height = height; } int64_t VulkanDriver::getStreamTimestamp(Handle sh) { @@ -1165,6 +1223,72 @@ int64_t VulkanDriver::getStreamTimestamp(Handle sh) { } void VulkanDriver::updateStreams(CommandStream* driver) { + FVK_SYSTRACE_SCOPE(); + if (UTILS_UNLIKELY(!mStreamsWithPendingAcquiredImage.empty())) { + for (auto& stream: mStreamsWithPendingAcquiredImage) { + if (stream->previousNeedsRelease()) { + scheduleRelease(stream->takePrevious()); + } + + // This executes on the backend thread (updateStreams is synchonous which means it + // executes on the user thread) Note: stream is captured by copy which is fine, this is + // a copy of a resource_ptr. We only need it find the associated stream + // inside the mStreamedImageManager texture bindings + driver->queueCommand([this, stream, s = stream.get(), + image = stream->getAcquired().image, + transform = stream->getFrontEndTransform()]() { + auto texture = s->getTexture(image); + s->setBackendTransform(transform); + if (!texture) { + auto externalImage = fvkutils::createExternalImageFromRaw(mPlatform, image, false); + auto metadata = mPlatform->extractExternalImageMetadata(externalImage); + auto imgData = mPlatform->createVkImageFromExternal(externalImage); + + assert_invariant(imgData.internal.valid() || imgData.external.valid()); + + VkFormat vkformat = metadata.format; + VkImage vkimage = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + if (imgData.internal.valid()) { + metadata.externalFormat = 0; + vkimage = imgData.internal.image; + memory = imgData.internal.memory; + } else { + vkformat = VK_FORMAT_UNDEFINED; + vkimage = imgData.external.image; + memory = imgData.external.memory; + } + + VkSamplerYcbcrConversion const conversion = + mExternalImageManager.getVkSamplerYcbcrConversion(metadata); + + auto newTexture = resource_ptr::construct(&mResourceManager, + mContext, mPlatform->getDevice(), mAllocator, &mResourceManager, + &mCommands, vkimage, memory, vkformat, conversion, metadata.samples, + metadata.width, metadata.height, metadata.layers, + metadata.filamentUsage, mStagePool); + + auto& commands = mCommands.get(); + // Unlike uploaded textures or swapchains, we need to explicit transition this + // texture into the read layout. + newTexture->transitionLayout(&commands, newTexture->getPrimaryViewRange(), + VulkanLayout::FRAG_READ); + + if (imgData.external.valid()) { + mExternalImageManager.addExternallySampledTexture(newTexture, + externalImage); + // Cache the AHB backed image. Acquires the image here. + s->pushImage(image, newTexture); + } + + texture = newTexture; + } + + mStreamedImageManager.onStreamAcquireImage(texture, stream); + }); + } + mStreamsWithPendingAcquiredImage.clear(); + } } void VulkanDriver::destroyFence(Handle fh) { @@ -1449,7 +1573,9 @@ void VulkanDriver::updateIndexBuffer(Handle ibh, BufferDescriptor void VulkanDriver::registerBufferObjectStreams(Handle boh, BufferObjectStreamDescriptor&& streams) { - // Noop + + auto bo = resource_ptr::cast(&mResourceManager, boh); + mStreamUniformDescriptors[bo.get()] = std::move(streams); } void VulkanDriver::updateBufferObject(Handle boh, BufferDescriptor&& bd, @@ -1458,6 +1584,34 @@ void VulkanDriver::updateBufferObject(Handle boh, BufferDescript auto bo = resource_ptr::cast(&mResourceManager, boh); commands.acquire(bo); + + if (UTILS_UNLIKELY(!mStreamUniformDescriptors.empty())) { + auto streamDescriptors = mStreamUniformDescriptors.find(bo.get()); + if (streamDescriptors != mStreamUniformDescriptors.end()) { + for (auto const& [offset, streamHandle, associationType]: + streamDescriptors->second.mStreams) { + if (associationType == BufferObjectStreamAssociationType::TRANSFORM_MATRIX) { + math::mat3f transform; + if (streamHandle) { + auto stream = + resource_ptr::cast(&mResourceManager, streamHandle); + if (stream->streamType == StreamType::NATIVE) { + FILAMENT_CHECK_PRECONDITION(false) + << "Native Stream not supported in Vulkan."; + } else { + // Backend call since getStreamTransformMatrix is called from async + // updateBufferObject. + transform = stream->getBackEndTransform(); + } + } + + copyMat3f(bd.buffer, offset, transform); + } + } + mStreamUniformDescriptors.erase(streamDescriptors); + } + } + bo->loadFromCpu(commands, bd.buffer, byteOffset, bd.size); scheduleDestroy(std::move(bd)); @@ -1525,6 +1679,12 @@ TimerQueryResult VulkanDriver::getTimerQueryValue(Handle tqh, uint } void VulkanDriver::setExternalStream(Handle th, Handle sh) { + FVK_SYSTRACE_SCOPE(); + auto t = resource_ptr::cast(&mResourceManager, th); + assert_invariant(t); + auto s = resource_ptr::cast(&mResourceManager, sh); + assert_invariant(s); + t->setStream(s); } void VulkanDriver::generateMipmaps(Handle th) { diff --git a/filament/backend/src/vulkan/VulkanDriver.h b/filament/backend/src/vulkan/VulkanDriver.h index 85ce2943b4e0..a1f6a4a95fe5 100644 --- a/filament/backend/src/vulkan/VulkanDriver.h +++ b/filament/backend/src/vulkan/VulkanDriver.h @@ -34,6 +34,7 @@ #include "vulkan/VulkanDescriptorSetCache.h" #include "vulkan/VulkanDescriptorSetLayoutCache.h" #include "vulkan/VulkanExternalImageManager.h" +#include "vulkan/VulkanStreamedImageManager.h" #include "vulkan/VulkanPipelineLayoutCache.h" #include "vulkan/memory/ResourceManager.h" #include "vulkan/memory/ResourcePointer.h" @@ -161,6 +162,12 @@ class VulkanDriver final : public DriverBase { VulkanDescriptorSetCache mDescriptorSetCache; VulkanQueryManager mQueryManager; VulkanExternalImageManager mExternalImageManager; + VulkanStreamedImageManager mStreamedImageManager; + + // Stream transforms + std::unordered_map mStreamUniformDescriptors; + math::mat3f getStreamTransformMatrix(Handle sh); + // This is necessary for us to write to push constants after binding a pipeline. using DescriptorSetLayoutHandleList = std::array, @@ -197,6 +204,10 @@ class VulkanDriver final : public DriverBase { bool const mIsSRGBSwapChainSupported; bool const mIsMSAASwapChainSupported; backend::StereoscopicType const mStereoscopicType; + + // setAcquiredImage is a DECL_DRIVER_API_SYNCHRONOUS_N which means we don't necessarily have the + // data to process it at call time. So we store it and process it during updateStreams. + std::vector> mStreamsWithPendingAcquiredImage; #if defined(__ANDROID__) AndroidProducerThrottling mProducerThrottling; #endif diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp index 18ecd0264258..585acb0f59e5 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.cpp +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.cpp @@ -31,9 +31,6 @@ namespace filament::backend { namespace { -using Bitmask = fvkutils::UniformBufferBitmask; -static_assert(sizeof(Bitmask) * 8 == fvkutils::MAX_DESCRIPTOR_SET_BITMASK_BITS); - template void erasep(std::vector& v, std::function f) { auto newEnd = std::remove_if(v.begin(), v.end(), f); @@ -50,31 +47,6 @@ ImageData& findImage(std::vector& images, return *itr; } -void copySet(VkDevice device, VkDescriptorSet srcSet, VkDescriptorSet dstSet, Bitmask bindings) { - // TODO: fix the size for better memory management - std::vector copies; - bindings.forEachSetBit([&](size_t index) { - copies.push_back({ - .sType = VK_STRUCTURE_TYPE_COPY_DESCRIPTOR_SET, - .srcSet = srcSet, - .srcBinding = (uint32_t) index, - .dstSet = dstSet, - .dstBinding = (uint32_t) index, - .descriptorCount = 1, - }); - }); - vkUpdateDescriptorSets(device, 0, nullptr, copies.size(), copies.data()); -} - -Bitmask foldBitsInHalf(Bitmask bitset) { - Bitmask outBitset; - bitset.forEachSetBit([&](size_t index) { - constexpr size_t BITMASK_LOWER_BITS_LEN = sizeof(outBitset) * 4; - outBitset.set(index % BITMASK_LOWER_BITS_LEN); - }); - return outBitset; -} - }// namespace VulkanExternalImageManager::VulkanExternalImageManager(VulkanPlatform* platform, @@ -172,38 +144,13 @@ void VulkanExternalImageManager::updateSetAndLayout( std::for_each(samplerAndBindings.begin(), samplerAndBindings.end(), [&](auto const& b) { outSamplers.push_back(std::get<1>(b)); }); - VkDescriptorSetLayout const oldLayout = layout->getExternalSamplerVkLayout(); VkDescriptorSetLayout const newLayout = mDescriptorSetLayoutCache->getVkLayout(layout->bitmask, actualExternalSamplers, outSamplers); - - // Need to copy the set - VkDescriptorSet const oldSet = set->getExternalSamplerVkSet(); - if (oldLayout != newLayout || oldSet == VK_NULL_HANDLE) { - // Build a new descriptor set from the new layout - VkDescriptorSet const newSet = mDescriptorSetCache->getVkSet(layout->count, newLayout); - auto const ubo = layout->bitmask.ubo | layout->bitmask.dynamicUbo; - auto const samplers = layout->bitmask.sampler & (~actualExternalSamplers); - - // Each bitmask denotes a binding index, and separated into two stages - vertex and buffer - // We fold the two stages into just the lower half of the bits to denote a combined set of - // bindings. - Bitmask const copyBindings = foldBitsInHalf(ubo | samplers); - VkDescriptorSet const srcSet = oldSet != VK_NULL_HANDLE ? oldSet : set->getVkSet(); - copySet(mPlatform->getDevice(), srcSet, newSet, copyBindings); - - set->setExternalSamplerVkSet(newSet, - [&descriptorSetCache = mDescriptorSetCache, layoutCount = layout->count, newLayout, - newSet](VulkanDescriptorSet*) { - descriptorSetCache->manualRecycle(layoutCount, newLayout, newSet); - }); - if (oldLayout != newLayout) { - layout->setExternalSamplerVkLayout(newLayout); - } - } - + layout->setExternalSamplerVkLayout(newLayout); // Update the external samplers in the set for (auto& [binding, sampler, image]: samplerAndBindings) { - mDescriptorSetCache->updateSamplerForExternalSamplerSet(set, binding, image); + mDescriptorSetCache->updateSampler(set, binding, image, VK_NULL_HANDLE /*sampler*/, + newLayout); } } @@ -261,7 +208,17 @@ void VulkanExternalImageManager::bindExternallySampledTexture( fvkmemory::resource_ptr image, SamplerParams samplerParams) { // Should we do duplicate validation here? auto& imageData = findImage(mImages, image); - mSetBindings.push_back({ bindingPoint, imageData.image, set, samplerParams }); + auto itr = std::find_if(mSetBindings.begin(), mSetBindings.end(), + [&](SetBindingInfo const& binding) { + return (binding.set == set && binding.binding == bindingPoint); + }); + if (itr == mSetBindings.end()) { + mSetBindings.push_back({ bindingPoint, imageData.image, set, samplerParams }); + } else { + // override the image data in the binding point + itr->image = image; + itr->samplerParams = samplerParams; + } } void VulkanExternalImageManager::addExternallySampledTexture( diff --git a/filament/backend/src/vulkan/VulkanExternalImageManager.h b/filament/backend/src/vulkan/VulkanExternalImageManager.h index 0f3173fc5ebf..399361532a3d 100644 --- a/filament/backend/src/vulkan/VulkanExternalImageManager.h +++ b/filament/backend/src/vulkan/VulkanExternalImageManager.h @@ -31,7 +31,7 @@ class VulkanSamplerCache; class VulkanDescriptorSetLayoutCache; class VulkanDescriptorSetCache; -// Manages the logic of external images and their quirks wrt Vulikan. +// Manages the logic of external images and their quirks wrt Vulkan. class VulkanExternalImageManager { public: diff --git a/filament/backend/src/vulkan/VulkanHandles.cpp b/filament/backend/src/vulkan/VulkanHandles.cpp index c2eef217a48b..5b72ae402020 100644 --- a/filament/backend/src/vulkan/VulkanHandles.cpp +++ b/filament/backend/src/vulkan/VulkanHandles.cpp @@ -40,6 +40,11 @@ namespace filament::backend { namespace { +// We don't have a good estimate for this magic number. This is used to remove empty +// slots from a descriptor set (a VulkanDescriptorset is backed by multiple +// VkDescriptorSet. +size_t DESCRIPTOR_SET_GC_LIMIT = 10; + inline VulkanBufferBinding getBufferObjectBinding(BufferObjectBinding bindingType) noexcept { switch (bindingType) { case BufferObjectBinding::VERTEX: @@ -192,15 +197,70 @@ VulkanDescriptorSetLayout::Bitmask VulkanDescriptorSetLayout::Bitmask::fromLayou return fromBackendLayout(layout); } -// This method will store an age associated with this command buffer into the VulkanBuffer, which -// will allow us to determine whether a barrier is necessary or not. +VulkanDescriptorSet::~VulkanDescriptorSet() { + for (auto setBundle: mSets) { + if (setBundle.onRecycleFn) { + setBundle.onRecycleFn(this); + } + } +} + +VulkanDescriptorSet::VulkanDescriptorSet(fvkmemory::resource_ptr layout, + OnRecycle&& onRecycleFn, VkDescriptorSet vkSet) + : dynamicUboMask(layout->bitmask.dynamicUbo), + uniqueDynamicUboCount(layout->count.dynamicUbo), + mLayout(layout), + mCurrentSetIndex(0) { + addNewSet(vkSet, std::move(onRecycleFn)); +} + void VulkanDescriptorSet::referencedBy(VulkanCommandBuffer& commands) { + // This will store an age associated with this command buffer into the VulkanBuffer, which + // will allow us to determine whether a barrier is necessary or not. mUboMask.forEachSetBit([this, &commands](size_t index) { auto& res = mResources[index]; fvkmemory::resource_ptr bo = fvkmemory::resource_ptr::cast((VulkanBufferObject*) res.get()); bo->referencedBy(commands); }); + mSets[mCurrentSetIndex].fenceStatus = commands.getFenceStatus(); +} + +void VulkanDescriptorSet::gc() { + size_t empty = 0; + for (auto& setBundle: mSets) { + assert_invariant(setBundle.onRecycleFn); + if (setBundle.vkSet == VK_NULL_HANDLE) { + empty++; + continue; + } + if (setBundle.fenceStatus && setBundle.fenceStatus->getStatus() == VK_SUCCESS) { + setBundle.onRecycleFn(this); + setBundle.vkSet = VK_NULL_HANDLE; + setBundle.fenceStatus = {}; + empty++; + } + } + + if (empty > DESCRIPTOR_SET_GC_LIMIT) { + std::vector retainedSets; + for (auto& setBundle: mSets) { + if (setBundle.vkSet != VK_NULL_HANDLE) { + retainedSets.push_back({ + setBundle.vkSet, + std::move(setBundle.onRecycleFn), + std::move(setBundle.fenceStatus) + }); + } + } + std::swap(mSets, retainedSets); + } +} + +void VulkanDescriptorSet::addNewSet(VkDescriptorSet vkSet, OnRecycle&& onRecycleFn) { + gc(); + mCurrentSetIndex = mSets.size(); + mSets.push_back({ vkSet, std::move(onRecycleFn) }); } PushConstantDescription::PushConstantDescription(backend::Program const& program) { diff --git a/filament/backend/src/vulkan/VulkanHandles.h b/filament/backend/src/vulkan/VulkanHandles.h index e185d750be26..50277639f14d 100644 --- a/filament/backend/src/vulkan/VulkanHandles.h +++ b/filament/backend/src/vulkan/VulkanHandles.h @@ -28,6 +28,7 @@ #include "VulkanTexture.h" #include "vulkan/VulkanCommands.h" #include "vulkan/memory/Resource.h" +#include "vulkan/memory/ResourcePointer.h" #include "vulkan/utils/Definitions.h" #include "vulkan/utils/StaticVector.h" @@ -57,6 +58,8 @@ inline uint8_t collapsedCount(Bitmask const& mask) { } // anonymous namespace +class VulkanDescriptorSetCache; + struct VulkanBufferObject; struct VulkanDescriptorSetLayout : public HwDescriptorSetLayout, fvkmemory::Resource { @@ -159,38 +162,14 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { using OnRecycle = std::function; VulkanDescriptorSet( - fvkutils::UniformBufferBitmask const& dynamicUboMask, - uint8_t uniqueDynamicUboCount, - OnRecycle&& onRecycleFn, VkDescriptorSet vkSet) - : dynamicUboMask(dynamicUboMask), - uniqueDynamicUboCount(uniqueDynamicUboCount), - mVkSet(vkSet), - mOnRecycleFn(std::move(onRecycleFn)) {} + fvkmemory::resource_ptr layout, + OnRecycle&& onRecycleFn, VkDescriptorSet vkSet); // NOLINTNEXTLINE(bugprone-exception-escape) - ~VulkanDescriptorSet() { - if (mOnRecycleFn) { - mOnRecycleFn(this); - } - if (mOnRecycleExternalSamplerFn) { - mOnRecycleExternalSamplerFn(this); - } - } + ~VulkanDescriptorSet(); VkDescriptorSet getVkSet() const noexcept { - return mVkSet; - } - - VkDescriptorSet getExternalSamplerVkSet() const noexcept { - return mExternalSamplerVkSet; - } - - void setExternalSamplerVkSet(VkDescriptorSet vkset, OnRecycle onRecycle) { - mExternalSamplerVkSet = vkset; - if (mOnRecycleExternalSamplerFn) { - mOnRecycleExternalSamplerFn(this); - } - mOnRecycleExternalSamplerFn = onRecycle; + return mSets[mCurrentSetIndex].vkSet; } void setOffsets(backend::DescriptorSetOffsetArray&& offsets) noexcept { @@ -211,17 +190,33 @@ struct VulkanDescriptorSet : public HwDescriptorSet, fvkmemory::Resource { void referencedBy(VulkanCommandBuffer& commands); - fvkutils::UniformBufferBitmask const dynamicUboMask; + bool isBound() const { + return bool(mSets[mCurrentSetIndex].fenceStatus); + } + + fvkmemory::resource_ptr getLayout() const { return mLayout; } + + fvkutils::UniformBufferBitmask const& dynamicUboMask; uint8_t const uniqueDynamicUboCount; private: - VkDescriptorSet const mVkSet; - VkDescriptorSet mExternalSamplerVkSet = VK_NULL_HANDLE; + friend class VulkanDescriptorSetCache; + + void addNewSet(VkDescriptorSet vkSet, OnRecycle&& onRecycleFn); + + void gc(); + + struct InternalVkSet { + VkDescriptorSet vkSet = {}; + OnRecycle onRecycleFn; + std::shared_ptr fenceStatus; + }; + fvkmemory::resource_ptr mLayout; backend::DescriptorSetOffsetArray mOffsets; std::vector> mResources; - OnRecycle mOnRecycleFn; - OnRecycle mOnRecycleExternalSamplerFn; + uint8_t mCurrentSetIndex; + std::vector mSets; fvkutils::UniformBufferBitmask mUboMask; }; diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp new file mode 100644 index 000000000000..cf5a2ac543b9 --- /dev/null +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "VulkanStreamedImageManager.h" + +#include "VulkanDescriptorSetCache.h" +#include "VulkanExternalImageManager.h" +#include "VulkanDescriptorSetLayoutCache.h" +#include "VulkanSamplerCache.h" + +namespace filament::backend { +VulkanStreamedImageManager::VulkanStreamedImageManager(VulkanExternalImageManager* manager) + : mExternalImageManager(manager) {} + +VulkanStreamedImageManager::~VulkanStreamedImageManager() = default; + +void VulkanStreamedImageManager::terminate() { mStreamedTexturesBindings.clear(); } + +void VulkanStreamedImageManager::bindStreamedTexture( + fvkmemory::resource_ptr set, + uint8_t bindingPoint, fvkmemory::resource_ptr image, + SamplerParams samplerParams) { + mStreamedTexturesBindings.push_back({ bindingPoint, image, set, samplerParams }); +} + +void VulkanStreamedImageManager::unbindStreamedTexture( + fvkmemory::resource_ptr set, + uint8_t bindingPoint) { + auto iter = std::remove_if(mStreamedTexturesBindings.begin(), mStreamedTexturesBindings.end(), + [&](StreamedTextureBinding& binding) { + return ((binding.set == set) && (binding.binding == bindingPoint)); + }); + mStreamedTexturesBindings.erase(iter, mStreamedTexturesBindings.end()); +} + +void VulkanStreamedImageManager::onStreamAcquireImage(fvkmemory::resource_ptr image, + fvkmemory::resource_ptr stream) { + for (StreamedTextureBinding const& data: mStreamedTexturesBindings) { + // Find the right stream + if (data.image->getStream() == stream) { + mExternalImageManager->bindExternallySampledTexture(data.set, data.binding, image, + data.samplerParams); + } + } +} + +} // namespace filament::backend diff --git a/filament/backend/src/vulkan/VulkanStreamedImageManager.h b/filament/backend/src/vulkan/VulkanStreamedImageManager.h new file mode 100644 index 000000000000..718e9693203a --- /dev/null +++ b/filament/backend/src/vulkan/VulkanStreamedImageManager.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TNT_FILAMENT_BACKEND_CACHING_VULKANSTREAMEDIMAGEMANAGER_H +#define TNT_FILAMENT_BACKEND_CACHING_VULKANSTREAMEDIMAGEMANAGER_H + +#include "VulkanHandles.h" + +#include + +#include +#include + +namespace filament::backend { + +class VulkanExternalImageManager; +class VulkanDescriptorSetCache; +class VulkanSamplerCache; + +// Manages the logic of streamed images. +class VulkanStreamedImageManager { +public: + VulkanStreamedImageManager( + VulkanExternalImageManager* manager); + ~VulkanStreamedImageManager(); + void terminate(); + +public: + void bindStreamedTexture(fvkmemory::resource_ptr set, uint8_t bindingPoint, + fvkmemory::resource_ptr image, SamplerParams samplerParams); + void unbindStreamedTexture(fvkmemory::resource_ptr set, + uint8_t bindingPoint); + void onStreamAcquireImage(fvkmemory::resource_ptr image, + fvkmemory::resource_ptr stream); + +private: + struct StreamedTextureBinding { + uint8_t binding = 0; + fvkmemory::resource_ptr image; + fvkmemory::resource_ptr set; + SamplerParams samplerParams; + }; + // keep track of all the stream bindings + std::vector mStreamedTexturesBindings; + + VulkanExternalImageManager* mExternalImageManager; +}; + +} // namespace filament::backend + +#endif // TNT_FILAMENT_BACKEND_CACHING_VULKANSTREAMEDIMAGEMANAGER_H diff --git a/filament/backend/src/vulkan/VulkanTexture.h b/filament/backend/src/vulkan/VulkanTexture.h index 02c88861d0ce..688e1c45b713 100644 --- a/filament/backend/src/vulkan/VulkanTexture.h +++ b/filament/backend/src/vulkan/VulkanTexture.h @@ -36,6 +36,49 @@ namespace filament::backend { struct VulkanTexture; +struct VulkanStream : public HwStream, fvkmemory::Resource { + + //-- These methods are only called from the frontend + void acquire(const AcquiredImage& image) { + user_thread.mPrevious = user_thread.mAcquired; + user_thread.mAcquired = image; + } + bool previousNeedsRelease() const { return (user_thread.mPrevious.image != nullptr); } + // this function will null the previous once the caller takes it. + // It ensures we don't schedule for release twice. + AcquiredImage takePrevious() { + AcquiredImage previous = user_thread.mPrevious; + user_thread.mPrevious = {}; + return previous; + } + const AcquiredImage& getAcquired() const { return user_thread.mAcquired; } + void setFrontEndTransform(const math::mat3f& transform) { user_thread.mTransform = transform; } + const math::mat3f& getFrontEndTransform() const { return user_thread.mTransform; } + + //-- These methods are only called from the backend thread + void setBackendTransform(const math::mat3f& transform) { mTransform = transform; } + const math::mat3f& getBackEndTransform() const { return mTransform; } + fvkmemory::resource_ptr getTexture(void* ahb) { + if (auto itr = mTextures.find(ahb); itr != mTextures.end()) { + return itr->second; + } + return {}; + } + void pushImage(void* ahb, fvkmemory::resource_ptr tex) { mTextures[ahb] = tex; } + +private: + // These are only called from the frontend + struct { + AcquiredImage mAcquired; + AcquiredImage mPrevious; + math::mat3f mTransform; + } user_thread; + + // #TODO b/442937292 + math::mat3f mTransform; + std::unordered_map> mTextures; +}; + struct VulkanTextureState : public fvkmemory::Resource { VulkanTextureState(VulkanStagePool& stagePool, VulkanCommands* commands, VmaAllocator allocator, VkDevice device, VkImage image, VkDeviceMemory deviceMemory, VkFormat format, @@ -74,6 +117,8 @@ struct VulkanTextureState : public fvkmemory::Resource { // The texture with the sidecar owns the sidecar. fvkmemory::resource_ptr mSidecarMSAA; + // The stream this texture is associated with. + fvkmemory::resource_ptr mStream; VkImage const mTextureImage; VkDeviceMemory const mTextureImageMemory; @@ -105,6 +150,7 @@ struct VulkanTextureState : public fvkmemory::Resource { using ImageViewHash = utils::hash::MurmurHashFn; std::unordered_map mCachedImageViews; + friend struct VulkanTexture; }; @@ -182,6 +228,14 @@ struct VulkanTexture : public HwTexture, fvkmemory::Resource { return mState->mSidecarMSAA; } + void setStream(fvkmemory::resource_ptr stream) { + mState->mStream = stream; + } + + fvkmemory::resource_ptr getStream() const { + return mState->mStream; + } + bool isTransientAttachment() const { return mState->mUsage & VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT; } diff --git a/filament/backend/src/vulkan/memory/Resource.cpp b/filament/backend/src/vulkan/memory/Resource.cpp index 72a2dbdc3234..e3381ca690e9 100644 --- a/filament/backend/src/vulkan/memory/Resource.cpp +++ b/filament/backend/src/vulkan/memory/Resource.cpp @@ -41,6 +41,7 @@ template ResourceType getTypeEnum() noexcept; template ResourceType getTypeEnum() noexcept; template ResourceType getTypeEnum() noexcept; template ResourceType getTypeEnum() noexcept; +template ResourceType getTypeEnum() noexcept; template ResourceType getTypeEnum() noexcept { @@ -104,6 +105,9 @@ ResourceType getTypeEnum() noexcept { if constexpr (std::is_same_v) { return ResourceType::SEMAPHORE; } + if constexpr (std::is_same_v) { + return ResourceType::STREAM; + } return ResourceType::UNDEFINED_TYPE; } @@ -149,6 +153,8 @@ std::string_view getTypeStr(ResourceType type) { return "VulkanMemoryMappedBuffer"; case ResourceType::SEMAPHORE: return "Semaphore"; + case ResourceType::STREAM: + return "VulkanStream"; case ResourceType::UNDEFINED_TYPE: return ""; } diff --git a/filament/backend/src/vulkan/memory/Resource.h b/filament/backend/src/vulkan/memory/Resource.h index 20d0302260ce..8d7b9e964a37 100644 --- a/filament/backend/src/vulkan/memory/Resource.h +++ b/filament/backend/src/vulkan/memory/Resource.h @@ -55,7 +55,8 @@ enum class ResourceType : uint8_t { SYNC = 17, MEMORY_MAPPED_BUFFER = 18, SEMAPHORE = 19, - UNDEFINED_TYPE = 20, // Must be the last enum because we use it for iterating over the enums. + STREAM = 20, + UNDEFINED_TYPE = 21, // Must be the last enum because we use it for iterating over the enums. }; template diff --git a/filament/backend/src/vulkan/memory/ResourceManager.cpp b/filament/backend/src/vulkan/memory/ResourceManager.cpp index 96f586dcef3f..94c4e84c61a4 100644 --- a/filament/backend/src/vulkan/memory/ResourceManager.cpp +++ b/filament/backend/src/vulkan/memory/ResourceManager.cpp @@ -123,6 +123,9 @@ void ResourceManager::destroyWithType(ResourceType type, HandleId id) { case ResourceType::SEMAPHORE: destruct(Handle(id)); break; + case ResourceType::STREAM: + destruct(Handle(id)); + break; case ResourceType::UNDEFINED_TYPE: break; } diff --git a/filament/backend/src/vulkan/utils/Image.cpp b/filament/backend/src/vulkan/utils/Image.cpp index 5bea3009c741..b613bc95bc30 100644 --- a/filament/backend/src/vulkan/utils/Image.cpp +++ b/filament/backend/src/vulkan/utils/Image.cpp @@ -18,6 +18,12 @@ #include "vulkan/VulkanTexture.h" +#if defined(__ANDROID__) +#include "backend/platforms/VulkanPlatformAndroid.h" +#include +#include +#endif + #include #include #include @@ -234,6 +240,19 @@ uint8_t reduceSampleCount(uint8_t sampleCount, VkSampleCountFlags mask) { return mostSignificantBit((sampleCount - 1) & mask); } +filament::backend::Platform::ExternalImageHandle createExternalImageFromRaw( + filament::backend::VulkanPlatform* platform, void* image, bool sRGB) { + +#if defined(__ANDROID__) + if (__builtin_available(android 26, *)) { + return static_cast(platform) + ->createExternalImage(reinterpret_cast(image), sRGB); + } +#endif + return {}; + +} + } // namespace filament::backend::fvkutils bool operator<(const VkImageSubresourceRange& a, const VkImageSubresourceRange& b) { diff --git a/filament/backend/src/vulkan/utils/Image.h b/filament/backend/src/vulkan/utils/Image.h index 5be3cd3769ab..19762a8bee22 100644 --- a/filament/backend/src/vulkan/utils/Image.h +++ b/filament/backend/src/vulkan/utils/Image.h @@ -18,6 +18,7 @@ #define TNT_FILAMENT_BACKEND_VULKAN_UTILS_IMAGE_H #include +#include #include @@ -106,6 +107,9 @@ VkImageAspectFlags getImageAspect(VkFormat format); uint8_t reduceSampleCount(uint8_t sampleCount, VkSampleCountFlags mask); +Platform::ExternalImageHandle createExternalImageFromRaw(filament::backend::VulkanPlatform* platform, + void* image, + bool sRGB); } // namespace fvkutils } // namespace filament::backend