How to feed cudaArray to Windows-Machine-Learning inference engine? - c++

Question

I am trying to develop an ML powered plugin for a real-time image processing software, that provides image data as cudaArray_t on the GPU, but because the software locks me into an older CUDA version, I would like to do this with DirectML (the software is Windows only anyways).
For latency reasons, I don't want to do any unnecessary GPU-CPU-GPU roundtrips. To do this, I thought that I would need to map the CUDA data to D3D12 resources, which then can be used to create input and output tensors to bind to the model. I have found a sample that uses the CUDA External Resource Interoperability API to map a cudaArray_t into a ID3D12Resource here that I am trying to base my code on. As I don't need to render anything, I thought I was able to simply create the heap and resource and then copy the incoming cudaArray_tinto the interop cudaArray_t as shown below, without needing to create any sort of command queue. Note that the missing code is the same as in the linked github repo above, so I left it out for conciseness.
This approach does not work, but I am not sure how to debug this, as I am new to Direct3D programming and GPU programming in general. I am using the official Direct3D 12 docs as a reference, but it is a bit overwhelming, so some direction on what should be fixed here would be greatly appreciated :) I was thinking that I need to use a semaphore for some kind of syncing, but I am not sure if that works without creating some sort of command queue.
bool initD3d12() {
// setup the d3d12 device
UINT dxgiFactoryFlags = 0;
winrt::com_ptr<IDXGIFactory4> factory;
winrt::check_hresult(CreateDXGIFactory2(dxgiFactoryFlags, IID_PPV_ARGS(factory.put())));
winrt::com_ptr<IDXGIAdapter1> hardwareAdapter;
GetHardwareAdapter(factory.get(), hardwareAdapter.put());
winrt::check_hresult(D3D12CreateDevice(hardwareAdapter.get(), D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(m_d3d12Device.put())));
DXGI_ADAPTER_DESC1 desc;
hardwareAdapter->GetDesc1(&desc);
m_dx12deviceluid = desc.AdapterLuid;
return true;
}
void initCuda() {
// setup the cuda device
int num_cuda_devices = 0;
checkCudaErrors(cudaGetDeviceCount(&num_cuda_devices));
if (!num_cuda_devices) {
throw std::exception("No CUDA Devices found");
}
for (int devId = 0; devId < num_cuda_devices; devId++) {
cudaDeviceProp devProp;
checkCudaErrors(cudaGetDeviceProperties(&devProp, devId));
if ((memcmp(&m_dx12deviceluid.LowPart, devProp.luid,
sizeof(m_dx12deviceluid.LowPart)) == 0) &&
(memcmp(&m_dx12deviceluid.HighPart,
devProp.luid + sizeof(m_dx12deviceluid.LowPart),
sizeof(m_dx12deviceluid.HighPart)) == 0)) {
checkCudaErrors(cudaSetDevice(devId));
m_cudaDeviceID = devId;
m_nodeMask = devProp.luidDeviceNodeMask;
checkCudaErrors(cudaStreamCreate(&m_streamToRun));
printf("CUDA Device Used [%d] %s\n", devId, devProp.name);
break;
}
}
}
void copyArrayToResource(cudaArray_t cudaArray) {
// then we want to copy cudaArray to the D3D texture, via its mapped form : cudaArray
cudaMemcpy2DArrayToArray(
m_cudaArray, // dst array
0, 0, // offset
cudaArray, 0, 0, // src
m_width * 4 * sizeof(float), m_height, // extent
cudaMemcpyDeviceToDevice); // kind
}
void createResource(size_t width, size_t height, ID3D12Resource** d3d12Resource) {
// Create a d3d12 resource in the desired size and map it to a cudaArray
m_width = width;
m_height = height;
// Create D3D12 2DTexture
// Assume 32-Bit float RGBA image
const auto channels = 4;
const auto textureSurface = width * height;
const auto texturePixels = textureSurface * channels;
const auto textureSizeBytes = sizeof(float)* texturePixels;
const auto texFormat = channels == 4 ? DXGI_FORMAT_R32G32B32A32_FLOAT : DXGI_FORMAT_R32G32B32_FLOAT;
const auto texDesc = CD3DX12_RESOURCE_DESC::Tex2D(texFormat, width, height, 1, 1, 1, 0, D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS);
D3D12_HEAP_PROPERTIES heapProperties = {
D3D12_HEAP_TYPE_DEFAULT,
D3D12_CPU_PAGE_PROPERTY_UNKNOWN,
D3D12_MEMORY_POOL_UNKNOWN,
0,
0};
winrt::check_hresult(m_d3d12Device->CreateCommittedResource(
&heapProperties,
D3D12_HEAP_FLAG_SHARED,
&texDesc,
D3D12_RESOURCE_STATE_COMMON,
nullptr,
IID_PPV_ARGS(d3d12Resource)));
// Create CUDA external resource
HANDLE sharedHandle;
WindowsSecurityAttributes windowsSecurityAttributes{};
LPCWSTR name = NULL;
winrt::check_hresult(m_d3d12Device->CreateSharedHandle(
*d3d12Resource, &windowsSecurityAttributes, GENERIC_ALL, 0,
&sharedHandle));
D3D12_RESOURCE_ALLOCATION_INFO d3d12ResourceAllocationInfo;
d3d12ResourceAllocationInfo = m_d3d12Device->GetResourceAllocationInfo(
m_nodeMask, 1, &texDesc);
size_t actualSize = d3d12ResourceAllocationInfo.SizeInBytes;
size_t alignment = d3d12ResourceAllocationInfo.Alignment;
cudaExternalMemoryHandleDesc externalMemoryHandleDesc;
memset(&externalMemoryHandleDesc, 0, sizeof(externalMemoryHandleDesc));
externalMemoryHandleDesc.type = cudaExternalMemoryHandleTypeD3D12Resource;
externalMemoryHandleDesc.handle.win32.handle = sharedHandle;
externalMemoryHandleDesc.size = actualSize;
externalMemoryHandleDesc.flags = cudaExternalMemoryDedicated;
checkCudaErrors(
cudaImportExternalMemory(&m_externalMemory, &externalMemoryHandleDesc));
cudaExternalMemoryMipmappedArrayDesc cuExtmemMipDesc{};
cuExtmemMipDesc.extent = make_cudaExtent(width, height, 0);
cuExtmemMipDesc.formatDesc = cudaCreateChannelDesc<float4>();
cuExtmemMipDesc.numLevels = 1;
cuExtmemMipDesc.flags = cudaArrayDefault;
cudaMipmappedArray_t cuMipArray{};
checkCudaErrors(cudaExternalMemoryGetMappedMipmappedArray(&cuMipArray, m_externalMemory, &cuExtmemMipDesc));
checkCudaErrors(cudaGetMipmappedArrayLevel(&m_cudaArray, cuMipArray, 0));
}
In the end if the mapping to a ID3D12Resource would work, I assume that one could use the ITensorStaticsNative interface to create a tensor to bind to the output or input of a LearningModel.