Stream API Vulkan Backend

filament/backend/CMakeLists.txt

@@ -204,6 +204,8 @@ if (FILAMENT_SUPPORTS_VULKAN)
             src/vulkan/VulkanDriverFactory.h
             src/vulkan/VulkanExternalImageManager.cpp
             src/vulkan/VulkanExternalImageManager.h
+            src/vulkan/VulkanStreamedImageManager.cpp
+            src/vulkan/VulkanStreamedImageManager.h
             src/vulkan/VulkanFboCache.cpp
             src/vulkan/VulkanFboCache.h
             src/vulkan/VulkanHandles.cpp

filament/backend/include/backend/Platform.h

@@ -70,6 +70,9 @@ class UTILS_PUBLIC Platform {
         ExternalImageHandle& operator=(ExternalImageHandle const& rhs) noexcept;
         ExternalImageHandle& operator=(ExternalImageHandle&& rhs) noexcept;
 
+        bool operator==(const ExternalImageHandle& rhs) const noexcept {
+            return mTarget == rhs.mTarget;
+        }
         explicit operator bool() const noexcept { return mTarget != nullptr; }
 
         ExternalImage* UTILS_NULLABLE get() noexcept { return mTarget; }

filament/backend/src/vulkan/VulkanDescriptorSetCache.cpp

@@ -19,6 +19,7 @@
 #include "VulkanCommands.h"
 #include "VulkanHandles.h"
 #include "VulkanConstants.h"
+#include "utils/compiler.h"
 
 #include <utils/FixedCapacityVector.h>
 #include <utils/Panic.h>
@@ -31,6 +32,18 @@ namespace filament::backend {
 
 namespace {
 
+using Bitmask = fvkutils::UniformBufferBitmask;
+static_assert(sizeof(Bitmask) * 8 == fvkutils::MAX_DESCRIPTOR_SET_BITMASK_BITS);
+
+Bitmask foldBitsInHalf(Bitmask bitset) {
+    Bitmask outBitset;
+    bitset.forEachSetBit([&](size_t index) {
+        constexpr size_t BITMASK_LOWER_BITS_LEN = sizeof(outBitset) * 4;
+        outBitset.set(index % BITMASK_LOWER_BITS_LEN);
+    });
+    return outBitset;
+}
+
 using DescriptorSetLayoutArray = VulkanDescriptorSetCache::DescriptorSetLayoutArray;
 using DescriptorCount = VulkanDescriptorSetCache::DescriptorCount;
 
@@ -304,8 +317,7 @@ void VulkanDescriptorSetCache::commit(VulkanCommandBuffer* commands,
         // This code actually binds the descriptor sets.
         auto set = updateSets[index];
         VkCommandBuffer const cmdbuffer = commands->buffer();
-        VkDescriptorSet vkset = useExternalSamplers[index] ? set->getExternalSamplerVkSet() :
-            set->getVkSet();
+        VkDescriptorSet const vkset = set->getVkSet();
         commands->acquire(set);
         vkCmdBindDescriptorSets(cmdbuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, index,
                 1, &vkset, set->uniqueDynamicUboCount, set->getOffsets()->data());
@@ -341,17 +353,39 @@ void VulkanDescriptorSetCache::updateBuffer(fvkmemory::resource_ptr<VulkanDescri
         .pBufferInfo = &info,
     };
     vkUpdateDescriptorSets(mDevice, 1, &descriptorWrite, 0, nullptr);
-
-    if (auto externalSamplerSet = set->getExternalSamplerVkSet();
-            externalSamplerSet != VK_NULL_HANDLE) {
-        descriptorWrite.dstSet = externalSamplerSet;
-        vkUpdateDescriptorSets(mDevice, 1, &descriptorWrite, 0, nullptr);
-    }
     set->acquire(bufferObject);
 }
 
-void VulkanDescriptorSetCache::updateSamplerImpl(VkDescriptorSet vkset, uint8_t binding,
-        fvkmemory::resource_ptr<VulkanTexture> texture, VkSampler sampler) noexcept {
+void VulkanDescriptorSetCache::updateSampler(fvkmemory::resource_ptr<VulkanDescriptorSet> set,
+        uint8_t binding, fvkmemory::resource_ptr<VulkanTexture> texture,
+        VkSampler sampler, VkDescriptorSetLayout externalSamplerLayout) noexcept {
+
+    // We have to update a bound set for two use cases
+    //   - streaming API (a changing feed of AHardwareBuffer)
+    //   - external samplers - potential changing of dataspace per-frame
+    if (UTILS_UNLIKELY(set->isBound())) {
+        auto layout = set->getLayout();
+        // Build a new descriptor set from the new layout
+        auto genLayout = externalSamplerLayout != VK_NULL_HANDLE ?
+                externalSamplerLayout : layout->getVkLayout();
+        VkDescriptorSet const newSet = getVkSet(layout->count, genLayout);
+        Bitmask const ubo = layout->bitmask.ubo | layout->bitmask.dynamicUbo;
+        Bitmask samplers = layout->bitmask.sampler;
+        samplers.unset(binding);
+
+        // Each bitmask denotes a binding index, and separated into two stages - vertex and buffer
+        // We fold the two stages into just the lower half of the bits to denote a combined set of
+        // bindings.
+        Bitmask const copyBindings = foldBitsInHalf(ubo | samplers);
+        VkDescriptorSet const srcSet = set->getVkSet();
+        copySet(srcSet, newSet, copyBindings);
+        set->addNewSet(newSet,
+                [this, layoutCount = layout->count, genLayout, newSet](VulkanDescriptorSet*) {
+                    this->manualRecycle(layoutCount, genLayout, newSet);
+                });
+    }
+
+    VkDescriptorSet const vkset = set->getVkSet();
     VkImageSubresourceRange range = texture->getPrimaryViewRange();
     VkImageViewType const expectedType = texture->getViewType();
     if (any(texture->usage & TextureUsage::DEPTH_ATTACHMENT) &&
@@ -376,19 +410,6 @@ void VulkanDescriptorSetCache::updateSamplerImpl(VkDescriptorSet vkset, uint8_t
         .pImageInfo = &info,
     };
     vkUpdateDescriptorSets(mDevice, 1, &descriptorWrite, 0, nullptr);
-}
-
-void VulkanDescriptorSetCache::updateSampler(fvkmemory::resource_ptr<VulkanDescriptorSet> set,
-        uint8_t binding, fvkmemory::resource_ptr<VulkanTexture> texture,
-        VkSampler sampler) noexcept {
-    updateSamplerImpl(set->getVkSet(), binding, texture, sampler);
-    set->acquire(texture);
-}
-
-void VulkanDescriptorSetCache::updateSamplerForExternalSamplerSet(
-        fvkmemory::resource_ptr<VulkanDescriptorSet> set, uint8_t binding,
-        fvkmemory::resource_ptr<VulkanTexture> texture) noexcept {
-    updateSamplerImpl(set->getExternalSamplerVkSet(), binding, texture, VK_NULL_HANDLE);
     set->acquire(texture);
 }
 
@@ -405,7 +426,7 @@ fvkmemory::resource_ptr<VulkanDescriptorSet> VulkanDescriptorSetCache::createSet
     auto const& count = layout->count;
     auto const vklayout = layout->getVkLayout();
     auto set = fvkmemory::resource_ptr<VulkanDescriptorSet>::make(
-            mResourceManager, handle, layout->bitmask.dynamicUbo, layout->count.dynamicUbo,
+            mResourceManager, handle, layout,
             [vkSet, count, vklayout, this](
                     VulkanDescriptorSet*) { this->manualRecycle(count, vklayout, vkSet); },
             vkSet);
@@ -428,4 +449,22 @@ void VulkanDescriptorSetCache::manualRecycle(VulkanDescriptorSetLayout::Count co
 
 void VulkanDescriptorSetCache::gc() { mStashedSets = {}; }
 
+void VulkanDescriptorSetCache::copySet(VkDescriptorSet srcSet, VkDescriptorSet dstSet,
+        fvkutils::SamplerBitmask bindings) const {
+    // TODO: fix the size for better memory management
+    std::vector<VkCopyDescriptorSet> copies;
+    bindings.forEachSetBit([&](size_t index) {
+        copies.push_back({
+            .sType = VK_STRUCTURE_TYPE_COPY_DESCRIPTOR_SET,
+            .srcSet = srcSet,
+            .srcBinding = (uint32_t) index,
+            .dstSet = dstSet,
+            .dstBinding = (uint32_t) index,
+            .descriptorCount = 1,
+        });
+    });
+    vkUpdateDescriptorSets(mDevice, 0, nullptr, copies.size(), copies.data());
+}
+
+
 } // namespace filament::backend

filament/backend/src/vulkan/VulkanDescriptorSetCache.h

@@ -55,7 +55,8 @@ class VulkanDescriptorSetCache {
             VkDeviceSize size) noexcept;
 
     void updateSampler(fvkmemory::resource_ptr<VulkanDescriptorSet> set, uint8_t binding,
-            fvkmemory::resource_ptr<VulkanTexture> texture, VkSampler sampler) noexcept;
+            fvkmemory::resource_ptr<VulkanTexture> texture, VkSampler sampler,
+            VkDescriptorSetLayout externalSamplerLayout = VK_NULL_HANDLE) noexcept;
 
     void updateSamplerForExternalSamplerSet(fvkmemory::resource_ptr<VulkanDescriptorSet> set, uint8_t binding,
             fvkmemory::resource_ptr<VulkanTexture> texture) noexcept;
@@ -90,8 +91,8 @@ class VulkanDescriptorSetCache {
     void resetCachedState() noexcept { mLastBoundInfo = {}; }
 
 private:
-    void updateSamplerImpl(VkDescriptorSet set, uint8_t binding,
-            fvkmemory::resource_ptr<VulkanTexture> texture, VkSampler sampler) noexcept;
+    void copySet(VkDescriptorSet srcSet, VkDescriptorSet destSet,
+            fvkutils::SamplerBitmask copyBindings) const;
 
     class DescriptorInfinitePool;