run time inconsistency DXVA hardware video decoding - c++

Question

I am currently working on a project that involves using DXVA API and the FFmpeg framework to implement hardware-accelerated decoding of H264 video stream files.
I have done some research on GPU decoding and constructed my code based on the hardware acceleration implementation in VLC. From my understanding, using DXVA in FFmpeg involves initializing the DirectXVideoDecoder and implementing several callback functions in AVCodecContext. The decoding process is done with the FFmpeg function avcodec_decode_video2() and each frame is parsed with av_read_frame(). The decoded frame is stored in the graphics memory and displayed using Direct3D.
I tried to time each process with :GetTickCount() function and noticed that the execution time of the program for a 1550 frame video is 35000ms, with the display function taking 90% of the time and decoding function taking 6% of the time.
However, when I tried to comment out the displaying process and execute the code only decoding each frame, the total decoding time surprisingly increased to 25,000ms for the same video, taking 94% of the total time.
Here is the code for the decoding function:
//record start time
DWORD start_time = ::GetTickCount();
//media file to be loaded
const char *filename = "123.mkv";
//time recording parameters
unsigned frame_read_time_total = 0;
unsigned decode_frame_time_total = 0;
unsigned display_time_total = 0;
unsigned setup_time_total = 0;
/*********************Setup and Initialization Code*******************************/
unsigned setup_time_start = ::GetTickCount();
av_register_all();
av_log_set_level(AV_LOG_DEBUG);
int res;
AVFormatContext *file = NULL;
res = avformat_open_input(&file, filename, NULL, NULL);//´ò¿ªÎļþ
if (res < 0) {
printf("error %x in avformat_open_input\n", res);
return 1;
}
res = avformat_find_stream_info(file, NULL);//È¡³öÁ÷ÐÅÏ¢
if (res < 0)
{
printf("error %x in avformat_find_stream_info\n", res);
return 1;
}
av_dump_format(file, 0, filename, 0);//ÁгöÊäÈëÎļþµÄÏà¹ØÁ÷ÐÅÏ¢
int i;
int videoindex = -1;
int audioindex = -1;
for (i = 0; i < file->nb_streams; i++){
if (file->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO){
videoindex = i;
}
if (file->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO){
audioindex = i;
}
}
if (videoindex == -1){
av_log(NULL, AV_LOG_DEBUG, "can't find video stream\n");
return 0;
}
AVCodec *codec = avcodec_find_decoder(file->streams[videoindex]->codec->codec_id);//¸ù¾ÝÁ÷ÐÅÏ¢ÕÒµ½½âÂëÆ÷
if (!codec){
printf("decoder not found\n");
return 1;
}
AVCodecContext *codecctx = file->streams[videoindex]->codec;
screen_width = codecctx->width;
screen_height = codecctx->height;
//Initialize Win API Window
WNDCLASSEX window;
ZeroMemory(&window, sizeof(window));
window.cbSize = sizeof(window);
window.hbrBackground = (HBRUSH)(COLOR_WINDOW + 1);
window.lpfnWndProc = (WNDPROC)WindowProcess;
window.lpszClassName = L"D3D";
window.style = CS_HREDRAW | CS_VREDRAW;
RegisterClassEx(&window);
HWND hwnd_temp = CreateWindow(L"D3D", L"Player", WS_OVERLAPPEDWINDOW,
0, 0, screen_width, screen_height, NULL, NULL, NULL, NULL);
if (hwnd_temp == NULL){
av_log(NULL, AV_LOG_ERROR, "Error: Cannot create window\n");
system("pause");
}
hwnd.push_back(hwnd_temp);
vlc_va_dxva2_t *dxva = vlc_va_NewDxva2(codecctx->codec_id);
if (NULL == dxva){
return 0;
}
res = Setup(dxva, &codecctx->hwaccel_context, &codecctx->pix_fmt, screen_width, screen_height);
if (res < 0) {
printf("error DXVA setup\n", res);
return 1;
}
//Assign callback function
codecctx->opaque = dxva;
codecctx->get_format = ffmpeg_GetFormat;
codecctx->get_buffer = ffmpeg_GetFrameBuf;
codecctx->reget_buffer = ffmpeg_ReGetFrameBuf;
codecctx->release_buffer = ffmpeg_ReleaseFrameBuf;
codecctx->thread_count = 1;
res = avcodec_open2(codecctx, codec, NULL);
if (res < 0) {
printf("error %x in avcodec_open2\n", res);
return 1;
}
//Initialize Packet
AVPacket pkt = { 0 };
AVFrame *picture = avcodec_alloc_frame();
DWORD wait_for_keyframe = 60;
//initialize frame count
int count = 0;
ShowWindow(hwnd.at(0), SW_SHOWNORMAL);
UpdateWindow(hwnd.at(0));
RECT screen_size;
screen_size.top = 0;
screen_size.bottom = screen_height;
screen_size.left = 0;
screen_size.right = screen_width;
unsigned setup_time_end = ::GetTickCount();
setup_time_total = setup_time_end - setup_time_start;
MSG msg;
ZeroMemory(&msg, sizeof(msg));
while(msg.message!=WM_QUIT)
{
if (PeekMessage(&msg, NULL, 0,0, PM_REMOVE)){
TranslateMessage(&msg);
DispatchMessage(&msg);
continue;
}
int read_status;
unsigned read_frame_start = ::GetTickCount();
read_status = av_read_frame(file, &pkt);
if (read_status < 0)
{
av_free_packet(&pkt);
goto done;
}
unsigned read_frame_end = ::GetTickCount();
frame_read_time_total += (read_frame_end - read_frame_start);
int got_picture = 0;
unsigned decode_start = ::GetTickCount();
int bytes_used = avcodec_decode_video2(codecctx, picture, &got_picture, &pkt);
unsigned decode_end = ::GetTickCount();
decode_frame_time_total += (decode_end - decode_start);
if (got_picture)
{
count++;
unsigned display_start = ::GetTickCount();
//display_frame((vlc_va_dxva2_t *)codecctx->opaque, picture, screen_size,0);
unsigned display_end = ::GetTickCount();
display_time_total += (display_end - display_start);
}
av_free_packet(&pkt);
}
done:
UnregisterClass(L"D3D",0);
printf("Frames = %d\n",count);
unsigned stop_time = ::GetTickCount();
unsigned total_time = stop_time - start_time;
printf("total frame = %d\n", count);
printf("time cost = %d\n", total_time);
printf("total setup time = %d, %f %% total execution time\n", setup_time_total,(float) setup_time_total / total_time * 100);
printf("total frame read time = %d, %f %% total execution time\n", frame_read_time_total, (float)frame_read_time_total / total_time*100);
printf("total frame decode time = %d, %f %% total execution time\n", decode_frame_time_total, (float)decode_frame_time_total / total_time*100);
printf("total display time = %d, %f %% of total execution time\n", display_time_total, (float)display_time_total / total_time*100);
av_free(picture);
av_close_input_file(file);
system("pause");
return 0;
What could be the cause of this strange behavior? My guess is that it may be the possible incorrect use of :GetTickCount() or may be it has to do with the DXVA hardware-accelerated decoding process. Sorry for the long post. Any input and suggestion is appreciated. Thanks in advance.

Answer 1

I think it is a correct behaviour, if the decoding process is asynchronous. I know Ffmpeg uses threads, but it depends on compilation flags or decoding setup.
If the display process is very long, the decoder decodes frames, while the display process executes. So when you ask for rendering, some frames are already decoded, and it's fast.
If you avoid the display process, the decoding process takes all the processor time. Normally, the display process uses some sort of timestamp that lets enough time to the decoding process.
PS : from what i know about Ffmpeg and Dxva2, you also need to provide the directx texture.