Skip to content

Work on property pool HLSL impl #649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 19 commits into
base: master
Choose a base branch
from
Draft

Work on property pool HLSL impl #649

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
a1747c6
Work on HLSL impl of property pools
deprilula28 Jan 17, 2024
adc4d57
Fix compilation issues with shader
deprilula28 Jan 19, 2024
d9ddf41
Nuke code that won't be used (for now)
deprilula28 Jan 21, 2024
1707158
Fix compilation problems
deprilula28 Jan 21, 2024
279c220
Temp fix for last compilation issue
deprilula28 Jan 21, 2024
4be1a3c
Fix implementation problems in HLSL and write initial transferPropert…
deprilula28 Jan 22, 2024
3570c03
Merge branch 'vulkan_1_3' into property_pool_hlsl
deprilula28 Jan 27, 2024
c44bb49
Fix merge build issues
deprilula28 Jan 27, 2024
9460e24
Fix vulkan_1_3 incompatibilities
deprilula28 Jan 27, 2024
88d1d00
Add temporary fill for testing
deprilula28 Jan 28, 2024
52d6972
WIP testing
deprilula28 Jan 29, 2024
706000d
Update to latest testing
deprilula28 Jan 30, 2024
b625153
Diff that breaks things
deprilula28 Jan 30, 2024
ef4b779
WIP Suballocated descriptor set
deprilula28 Feb 12, 2024
b8db8c9
Work on sub allocated descriptor set
deprilula28 Feb 13, 2024
1a0c998
Apply fixes to property pool stuff
deprilula28 Feb 14, 2024
99d80a7
PR review fixes
deprilula28 Feb 15, 2024
61604ee
More PR reviws
deprilula28 Feb 15, 2024
7ac728b
PR reviews
deprilula28 Feb 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 38 additions & 23 deletions include/nbl/builtin/hlsl/property_pool/copy.comp.hlsl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,20 @@ struct TransferLoop
const uint64_t srcIndexSize = uint64_t(1) << SrcIndexSizeLog2;
const uint64_t dstIndexSize = uint64_t(1) << DstIndexSizeLog2;

const uint64_t srcOffset = invocationIndex * srcIndexSize * transferRequest.propertySize;
const uint64_t dstOffset = invocationIndex * dstIndexSize * transferRequest.propertySize;
// Fill: Always use offset 0 on src
const uint64_t srcOffset = Fill ? 0 : invocationIndex * transferRequest.propertySize;
const uint64_t dstOffset = invocationIndex * transferRequest.propertySize;

const uint64_t srcIndexAddress = Fill ? transferRequest.srcIndexAddr + srcOffset : transferRequest.srcIndexAddr;
const uint64_t dstIndexAddress = Fill ? transferRequest.dstIndexAddr + dstOffset : transferRequest.dstIndexAddr;

const uint64_t srcAddressBufferOffset = SrcIndexIota ? srcIndexAddress : vk::RawBufferLoad<uint32_t>(srcIndexAddress);
const uint64_t dstAddressBufferOffset = DstIndexIota ? dstIndexAddress : vk::RawBufferLoad<uint32_t>(dstIndexAddress);
// IOTA: Use the index as the fetching offset
// Non IOTA: Read the address buffer ("index buffer") to select fetching offset
const uint64_t srcAddressBufferOffset = SrcIndexIota ? srcOffset : vk::RawBufferLoad<uint32_t>(transferRequest.srcIndexAddr + srcOffset * sizeof(uint32_t));
const uint64_t dstAddressBufferOffset = DstIndexIota ? dstOffset : vk::RawBufferLoad<uint32_t>(transferRequest.dstIndexAddr + dstOffset * sizeof(uint32_t));

const uint64_t srcAddressMapped = transferRequest.srcAddr + srcAddressBufferOffset * srcIndexSize;
const uint64_t dstAddressMapped = transferRequest.dstAddr + dstAddressBufferOffset * dstIndexSize;

//vk::RawBufferStore<uint64_t>(transferRequest.dstAddr + invocationIndex * sizeof(uint64_t) * 2, srcAddressMapped,8);
//vk::RawBufferStore<uint64_t>(transferRequest.dstAddr + invocationIndex * sizeof(uint64_t) * 2 + sizeof(uint64_t), dstAddressMapped,8);
if (SrcIndexSizeLog2 == 0) {} // we can't write individual bytes
else if (SrcIndexSizeLog2 == 1) vk::RawBufferStore<uint16_t>(dstAddressMapped, vk::RawBufferLoad<uint16_t>(srcAddressMapped));
else if (SrcIndexSizeLog2 == 2) vk::RawBufferStore<uint32_t>(dstAddressMapped, vk::RawBufferLoad<uint32_t>(srcAddressMapped));
Expand Down Expand Up @@ -111,36 +113,49 @@ void main(uint32_t3 dispatchId)

// Loading transfer request from the pointer (can't use struct
// with BDA on HLSL SPIRV)
uint64_t transferCmdAddr = globals.transferCommandsAddress + sizeof(TransferRequest) * propertyId;
TransferRequest transferRequest;
transferRequest.srcAddr = vk::RawBufferLoad<uint>(globals.transferCommandsAddress) | vk::RawBufferLoad<uint>(globals.transferCommandsAddress + sizeof(uint)) << 32;
transferRequest.dstAddr = vk::RawBufferLoad<uint64_t>(globals.transferCommandsAddress + sizeof(uint64_t));
transferRequest.srcIndexAddr = vk::RawBufferLoad<uint64_t>(globals.transferCommandsAddress + sizeof(uint64_t) * 2);
transferRequest.dstIndexAddr = vk::RawBufferLoad<uint64_t>(globals.transferCommandsAddress + sizeof(uint64_t) * 3);
transferRequest.srcAddr = vk::RawBufferLoad<uint64_t>(transferCmdAddr,8);
transferRequest.dstAddr = vk::RawBufferLoad<uint64_t>(transferCmdAddr + sizeof(uint64_t),8);
transferRequest.srcIndexAddr = vk::RawBufferLoad<uint64_t>(transferCmdAddr + sizeof(uint64_t) * 2,8);
transferRequest.dstIndexAddr = vk::RawBufferLoad<uint64_t>(transferCmdAddr + sizeof(uint64_t) * 3,8);
Comment on lines +126 to +129
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make a wrapper for vk::RawBufferLoad and Store in nbl::hlsl::legacy which uses our type traits to default the alignment

// Remaining elements are part of the same bitfield
// TODO: Do this only using raw buffer load?
uint64_t bitfieldType = vk::RawBufferLoad<uint64_t>(globals.transferCommandsAddress + sizeof(uint64_t) * 4);
uint64_t bitfieldType = vk::RawBufferLoad<uint64_t>(transferCmdAddr + sizeof(uint64_t) * 4,8);
transferRequest.elementCount32 = uint32_t(bitfieldType);
transferRequest.elementCountExtra = uint32_t(bitfieldType);
transferRequest.propertySize = uint32_t(bitfieldType >> 3);
transferRequest.fill = uint32_t(bitfieldType >> (3 + 24));
transferRequest.srcIndexSizeLog2 = uint32_t(bitfieldType >> (3 + 24 + 1));
transferRequest.dstIndexSizeLog2 = uint32_t(bitfieldType >> (3 + 24 + 1 + 2));
transferRequest.elementCountExtra = uint32_t(bitfieldType >> 32);
transferRequest.propertySize = uint32_t(bitfieldType >> (32 + 3));
transferRequest.fill = uint32_t(bitfieldType >> (32 + 3 + 24));
transferRequest.srcIndexSizeLog2 = uint32_t(bitfieldType >> (32 + 3 + 24 + 1));
transferRequest.dstIndexSizeLog2 = uint32_t(bitfieldType >> (32 + 3 + 24 + 1 + 2));

const uint dispatchSize = nbl::hlsl::device_capabilities_traits<device_capabilities>::maxOptimallyResidentWorkgroupInvocations;
const bool fill = transferRequest.fill == 1;

vk::RawBufferStore<uint64_t>(globals.transferCommandsAddress + 40 * 3, transferRequest.srcAddr);
vk::RawBufferStore<uint64_t>(globals.transferCommandsAddress + 40 * 4, transferRequest.dstAddr);
vk::RawBufferStore<uint>(globals.transferCommandsAddress + 40 * 5, vk::RawBufferLoad<uint>(transferRequest.srcAddr + sizeof(uint16_t) * 3));
//if (fill) { TransferLoopPermutationFill<true> loop; loop.copyLoop(invocationIndex, propertyId, transferRequest, dispatchSize); }
//else { TransferLoopPermutationFill<false> loop; loop.copyLoop(invocationIndex, propertyId, transferRequest, dispatchSize); }
//uint64_t debugWriteAddr = transferRequest.dstAddr + sizeof(uint64_t) * 9 * propertyId;
//vk::RawBufferStore<uint64_t>(debugWriteAddr + sizeof(uint64_t) * 0, transferRequest.srcAddr,8);
//vk::RawBufferStore<uint64_t>(debugWriteAddr + sizeof(uint64_t) * 1, transferRequest.dstAddr,8);
//vk::RawBufferStore<uint64_t>(debugWriteAddr + sizeof(uint64_t) * 2, transferRequest.srcIndexAddr,8);
//vk::RawBufferStore<uint64_t>(debugWriteAddr + sizeof(uint64_t) * 3, transferRequest.dstIndexAddr,8);
//uint64_t elementCount = uint64_t(transferRequest.elementCount32)
// | uint64_t(transferRequest.elementCountExtra) << 32;
//vk::RawBufferStore<uint64_t>(debugWriteAddr + sizeof(uint64_t) * 4, elementCount,8);
//vk::RawBufferStore<uint32_t>(debugWriteAddr + sizeof(uint64_t) * 5, transferRequest.propertySize,4);
//vk::RawBufferStore<uint32_t>(debugWriteAddr + sizeof(uint64_t) * 6, transferRequest.fill,4);
//vk::RawBufferStore<uint32_t>(debugWriteAddr + sizeof(uint64_t) * 7, transferRequest.srcIndexSizeLog2,4);
//vk::RawBufferStore<uint32_t>(debugWriteAddr + sizeof(uint64_t) * 8, transferRequest.dstIndexSizeLog2,4);
//vk::RawBufferStore<uint64_t>(transferRequest.dstAddr + sizeof(uint64_t) * invocationIndex, invocationIndex,8);

if (fill) { TransferLoopPermutationFill<true> loop; loop.copyLoop(invocationIndex, propertyId, transferRequest, dispatchSize); }
else { TransferLoopPermutationFill<false> loop; loop.copyLoop(invocationIndex, propertyId, transferRequest, dispatchSize); }
}

}
}
}

[numthreads(1,1,1)]
// TODO: instead use some sort of replace function for getting optimal size?
[numthreads(512,1,1)]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I already wrote on discord to codegen a 5 line compute shader with main in the C++ and leave copy.hlsl as stage agnostic

void main(uint32_t3 dispatchId : SV_DispatchThreadID)
{
nbl::hlsl::property_pools::main<nbl::hlsl::jit::device_capabilities>(dispatchId);
Expand Down
17 changes: 14 additions & 3 deletions src/nbl/video/utilities/CPropertyPoolHandler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,8 @@ bool CPropertyPoolHandler::transferProperties(
auto srcRequest = transferPassRequests + i;
transferRequest.srcAddr = srcRequest->memblock.buffer.get()->getDeviceAddress() + srcRequest->memblock.offset;
transferRequest.dstAddr = srcRequest->buffer.buffer.get()->getDeviceAddress() + srcRequest->buffer.offset;
transferRequest.srcIndexAddr = srcRequest->srcAddressesOffset ? addressBufferDeviceAddr + srcRequest->srcAddressesOffset : 0;
transferRequest.dstIndexAddr = srcRequest->dstAddressesOffset ? addressBufferDeviceAddr + srcRequest->dstAddressesOffset : 0;
transferRequest.srcIndexAddr = srcRequest->srcAddressesOffset != IPropertyPool::invalid ? addressBufferDeviceAddr + srcRequest->srcAddressesOffset : 0;
transferRequest.dstIndexAddr = srcRequest->dstAddressesOffset != IPropertyPool::invalid ? addressBufferDeviceAddr + srcRequest->dstAddressesOffset : 0;
transferRequest.elementCount32 = uint32_t(srcRequest->elementCount & (uint64_t(1) << 32) - 1);
transferRequest.elementCountExtra = uint32_t(srcRequest->elementCount >> 32);
transferRequest.propertySize = srcRequest->elementSize;
Expand All @@ -144,7 +144,18 @@ bool CPropertyPoolHandler::transferProperties(
maxElements = core::max<uint64_t>(maxElements, srcRequest->elementCount);
}
cmdbuf->updateBuffer({ scratch.offset,sizeof(TransferRequest) * requestsThisPass, core::smart_refctd_ptr(scratch.buffer) }, transferRequestsData);
// TODO: pipeline barrier

const asset::SMemoryBarrier barriers[1] = { {
.srcStageMask = asset::PIPELINE_STAGE_FLAGS::COPY_BIT,
.srcAccessMask = asset::ACCESS_FLAGS::TRANSFER_WRITE_BIT,
.dstStageMask = asset::PIPELINE_STAGE_FLAGS::COMPUTE_SHADER_BIT,
.dstAccessMask = asset::ACCESS_FLAGS::SHADER_READ_BITS
} };
cmdbuf->pipelineBarrier(asset::EDF_NONE,IGPUCommandBuffer::SPipelineBarrierDependencyInfo{
.memBarriers = barriers
// TODO: .bufBarriers = instead
});

cmdbuf->bindComputePipeline(m_pipeline.get());

nbl::hlsl::property_pools::GlobalPushContants pushConstants;
Expand Down