How to debug VK_ERROR_DEVICE_LOST from calling vkCmdTraceRaysNV? - c++

Question

I'm working on a vulkan ray tracing application and I'm kinda stuck. Everything seems to work up until I call vkCmdTraceRaysNV and record it into my command buffer, the application crashes with an exception in vkQueueSubmit returning VK_ERROR_DEVICE_LOST. I have checked basic stuff in Nvidia NSight, textures seem correct and acceleration structures are visualized correctly. Standard validation layer gives nothing. I think its either something with the shader binding table or the contents of the command buffer executing in a weird order. Are there any other options for validation besides staring at code?
This is the content of the command buffer, maybe someone knows whatsup (if I comment vkCmdTraceRaysNV out it doesnt crash):
VkViewport viewport{};
viewport.x = 0.0f;
viewport.y = 0.0f;
viewport.width = static_cast<float>(extent.width);
viewport.height = static_cast<float>(extent.height);
viewport.minDepth = 0.0f;
viewport.maxDepth = 1.0f;
VkRect2D scissor{};
scissor.offset = { 0, 0 };
scissor.extent = extent;
VkRenderPassBeginInfo renderPassInfo{};
renderPassInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
renderPassInfo.renderPass = renderPass;
renderPassInfo.framebuffer = framebuffers[framebufferIndex];
renderPassInfo.renderArea.offset = { 0, 0 };
renderPassInfo.renderArea.extent = extent;
std::array<VkClearValue, 2> clearValues{};
clearValues[0].color = { 0.0f, 0.0f, 0.0f, 1.0f };
clearValues[1].depthStencil = { 1.0f, 0 };
renderPassInfo.clearValueCount = static_cast<uint32_t>(clearValues.size());
renderPassInfo.pClearValues = clearValues.data();
// begin normal vertex rendering
vkCmdSetViewport(commandBuffer, 0, 1, &viewport);
vkCmdSetScissor(commandBuffer, 0, 1, &scissor);
vkCmdBeginRenderPass(commandBuffer, &renderPassInfo, VK_SUBPASS_CONTENTS_INLINE);
vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
VkBuffer vertexBuffers[] = { vertexBuffer };
VkDeviceSize offset[] = { 0 };
vkCmdBindVertexBuffers(commandBuffer, 0, 1, vertexBuffers, offset);
vkCmdBindIndexBuffer(commandBuffer, indexBuffer, 0, VK_INDEX_TYPE_UINT16);
vkCmdBindDescriptorSets(commandBuffer, VkPipelineBindPoint::VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &descriptorSet, 0, nullptr);
for (const auto& object : objects) {
vkCmdPushConstants(commandBuffer, pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(glm::mat4), &object.model);
vkCmdDrawIndexed(commandBuffer, static_cast<uint32_t>(object.indices.size()), 1, object.indexOffset, object.vertexOffset, 0);
}
vkCmdEndRenderPass(commandBuffer);
// acquire textures for ray tracing use
ImageMemoryBarrier(cmdBuffer, depthTexture.image, VK_IMAGE_ASPECT_DEPTH_BIT,
0, VK_ACCESS_SHADER_READ_BIT, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL);
ImageMemoryBarrier(cmdBuffer, shadowsTexture.image, VK_IMAGE_ASPECT_COLOR_BIT,
0, VK_ACCESS_SHADER_READ_BIT, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL);
// bind the pipeline and resources
vkCmdBindPipeline(cmdBuffer, VK_PIPELINE_BIND_POINT_RAY_TRACING_NV, pipeline);
vkCmdBindDescriptorSets(cmdBuffer, VK_PIPELINE_BIND_POINT_RAY_TRACING_NV, pipelineLayout, 0, 1, &descriptorSet, 0, nullptr);
vkCmdPushConstants(cmdBuffer, pipelineLayout, VK_SHADER_STAGE_RAYGEN_BIT_NV, 0, sizeof(pushData), &pushData);
// calculate SBT
VkDeviceSize progSize = rtProps.shaderGroupBaseAlignment; // Size of a program identifier
VkDeviceSize rayGenOffset = 0u * progSize; // Start at the beginning of m_sbtBuffer
VkDeviceSize missOffset = 1u * progSize; // Jump over raygen
VkDeviceSize missStride = progSize;
// run ray tracing, this is where it crashes
vk_nv_ray_tracing::vkCmdTraceRaysNV(cmdBuffer,
sbtBuffer, rayGenOffset, // raygen group
sbtBuffer, missOffset, missStride, // miss group
VK_NULL_HANDLE, 0, 0,
VK_NULL_HANDLE, 0, 0,
width, height, 1
);
Shader Binding Table code:
/// define the groups, a miss group and raygen group
VkRayTracingShaderGroupCreateInfoNV group = {};
group.generalShader = 0;
group.anyHitShader = VK_SHADER_UNUSED_NV;
group.closestHitShader = VK_SHADER_UNUSED_NV;
group.intersectionShader = VK_SHADER_UNUSED_NV;
group.sType = VkStructureType::VK_STRUCTURE_TYPE_RAY_TRACING_SHADER_GROUP_CREATE_INFO_NV;
group.type = VkRayTracingShaderGroupTypeNV::VK_RAY_TRACING_SHADER_GROUP_TYPE_GENERAL_NV;
groups.push_back(group);
group.generalShader = 1;
groups.push_back(group);
const uint32_t groupCount = static_cast<uint32_t>(groups.size());
const uint32_t sbtSize = groupCount * rtProps.shaderGroupBaseAlignment;
std::vector<uint8_t> shaderHandles(sbtSize);
if (vk_nv_ray_tracing::vkGetRayTracingShaderGroupHandlesNV(device, pipeline, 0, groupCount, sbtSize, shaderHandles.data()) != VK_SUCCESS) {
throw std::runtime_error("failed to get rt shader group handles");
}
VkBufferCreateInfo sbtBufferCreateInfo = {};
sbtBufferCreateInfo.size = sbtSize;
sbtBufferCreateInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
sbtBufferCreateInfo.sType = VkStructureType::VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
sbtBufferCreateInfo.usage = VkBufferUsageFlagBits::VK_BUFFER_USAGE_RAY_TRACING_BIT_NV;
VmaAllocationCreateInfo sbtBufferAllocInfo = {};
sbtBufferAllocInfo.usage = VMA_MEMORY_USAGE_CPU_TO_GPU;
sbtBufferAllocInfo.flags = VmaAllocationCreateFlagBits::VMA_ALLOCATION_CREATE_MAPPED_BIT;
VmaAllocationInfo allocInfo{};
if (vmaCreateBuffer(allocator, &sbtBufferCreateInfo, &sbtBufferAllocInfo, &sbtBuffer, &sbtAlloc, &allocInfo) != VK_SUCCESS) {
throw std::runtime_error("failed to create sbt buffer");
}
auto* pData = reinterpret_cast<uint8_t*>(allocInfo.pMappedData);
for (uint32_t g = 0; g < groupCount; g++) {
std::memcpy(pData, groups.data() + g * rtProps.shaderGroupHandleSize, rtProps.shaderGroupHandleSize);
pData += rtProps.shaderGroupBaseAlignment;
}

Answer 1

I had the same problem. Even though I didn't create the acceleration structure and I didn't bind data, it still returns VK_ERROR_DEVICE_LOST and VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT was reported in VK_NV_device_diagnostic_checkpoints
In my case, there was a problem with the size of the shader binding table and how it was copied.