MVAPICH deadlocks on CUDA memory while kernel is running - c++

Question

I try to get a MPI-CUDA program working with MVAPICH CUDA8. I did run the program successfully with openMPI before but I want to test if I get better performance with MVAPICH. Unfortunately the program gets stuck in MPI_Isend if a CUDA kernel is running at the same time when using MVAPICH.
I downloaded MVAPICH2-2.2 and built it from the source with the configuration flags
--enable-cuda --disable-mcast
to enable MPI calls on cuda memory. mcast was disabled because I could not compile it without the flag.
I used the following flags before running the application:
export MV2_USE_CUDA=1
export MV2_GPUDIRECT_GDRCOPY_LIB=/path/to/gdrcopy/
export MV2_USE_GPUDIRECT=1
MPI_Isend/recv work fine when at the same time no CUDA kernel is running. But in my program it is important that MPI sends and receives data from and to GPU memory, while a kernel is running.
I came up with two possible reasons for that behavior. First, MVAPICH tries to run his own CUDA kernel for some reason to send data from GPU memory and this kernel does not get scheduled because the GPU is already fully utilized. Second possibility: MVAPICH uses cudaMemcpy somewhere (not the async version), which blocks until the kernel finishes execution.
Could someone confirm one of my assumptions? And is there a flag in MVAPICH that solves this problem that I am not aware of?
EDIT:
Here a "simpel" code that illustrates my problem. When executing the code with openMPI, it executes and terminates correctly. With mvapich2 it deadlocks at the marked MPI_Send function.
#include <stdio.h>
#include <stdlib.h>
#include <cuda.h>
#include <mpi.h>
__global__ void kernel(double * buffer, int rank)
{
volatile double *buf = buffer;
if(rank == 0){
while(buf[0] != 3){}
} else {
while(buf[0] != 2){}
}
}
int main(int argc, char **argv)
{
double host_buffer[1];
MPI_Init(&argc, &argv);
int world_size, world_rank;
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
printf("Im rank %d\n", world_rank);
cudaSetDevice(world_rank);
double * dev_buffer;
cudaError_t err = cudaMalloc(&dev_buffer, sizeof(double));
if(world_rank == 0){
host_buffer[0] = 1;
cudaError_t err = cudaMemcpy(dev_buffer, host_buffer, sizeof(double), cudaMemcpyHostToDevice);
MPI_Send(dev_buffer, 1, MPI_DOUBLE, 1, 0, MPI_COMM_WORLD);
printf("[%d]First send does not deadlock\n", world_rank);
}else {
MPI_Recv(dev_buffer, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
printf("[%d]Received first message\n", world_rank);
}
cudaStream_t stream, kernel_stream;
cudaStreamCreate(&stream);
cudaStreamCreate(&kernel_stream);
printf("[%d]launching kernel\n", world_rank);
kernel<<<208, 128, 0, kernel_stream>>>(dev_buffer, world_rank);
if(world_rank == 0){
//rank 0
host_buffer[0] = 2;
cudaMemcpyAsync(
dev_buffer, host_buffer, sizeof(double),
cudaMemcpyHostToDevice,
stream
);
cudaStreamSynchronize(stream);
printf("[%d]Send message\n", world_rank);
MPI_Send(dev_buffer, 1, MPI_DOUBLE, 1, 0, MPI_COMM_WORLD); //mvapich2 deadlocks here
printf("[%d]Message sent\n", world_rank);
printf("[%d]Receive message\n", world_rank);
MPI_Recv(dev_buffer, 1, MPI_DOUBLE, 1, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
printf("[%d]Message received\n", world_rank);
cudaStreamSynchronize(kernel_stream);
printf("[%d]kernel finished\n", world_rank);
} else {
//rank 1
printf("[%d]Receive message\n", world_rank);
MPI_Recv(dev_buffer, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
printf("[%d]Message received\n", world_rank);
cudaStreamSynchronize(kernel_stream);
printf("[%d]kernel finished\n", world_rank);
host_buffer[0] = 3;
cudaMemcpyAsync(
dev_buffer, host_buffer, sizeof(double),
cudaMemcpyHostToDevice,
stream
);
cudaStreamSynchronize(stream);
printf("[%d]Send message\n", world_rank);
MPI_Send(dev_buffer, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD);
printf("[%d]Message sent\n", world_rank);
}
printf("[%d]Stopped execution\n", world_rank);
MPI_Finalize();
}

Answer 1

I got back to this problem and used gdb to debug the code.
Apparently, the problem is the eager protocol of MVAPICH2 implemented in src/mpid/ch3/channels/mrail/src/gen2/ibv_send.c. The eager protocol uses a cuda_memcpy without async, which blocks until the kernel execution finishes.
The program posted in the question runs fine by passing MV2_IBA_EAGER_THRESHOLD 1 to mpirun. This prevents MPI to use the eager protocol and uses the rendez-vous protocol instead.
Patching the MVAPICH2 source code does solve the problem as well. I changed the synchronous cudaMemcpys to cudaMemcpyAsync in the files
src/mpid/ch3/channels/mrail/src/gen2/ibv_send.c
src/mpid/ch3/channels/mrail/src/gen2/ibv_recv.c
src/mpid/ch3/src/ch3u_request.c
The change in the third file is only needed for MPI_Isend/MPI_Irecv. Other MPI functions might need some additional code changes.