Capture from desktop (VFR) to mp4 - how to deal with frame rate? - c++

Question

I'm putting frames from the Desktop Duplication API through a Media Foundation H264 encoder transform (in this case, NVIDIA's hardware accelerated encoder) and then into a SinkWriter to put the frames into an MP4 container. This whole process works quite well and is extremely fast.
The problem
The video is in slow motion, so to speak. I have the frame rates set everywhere to 60 fps, and the sample time and durations are based on that 60 fps. What is happening is that I am providing way more frames than 60 per second to the SinkWriter. I don't quite know how the MP4 format and video players work but I assume it is simply looking at the frame duration (16.7ms for 60fps) and since there are way more frames than 60, it appears slowed down.
I've had a very hard time finding out how to 'properly' limit the frames being provided to the encoder/sink writer. If I simply Sleep for 15ms or so, the video appears fine but I realize that's not the way to do it - it was just for testing and to confirm my theory. I've tried using the Frame Rate Converter DSP but it gives me an E_UNEXPECTED because I don't think it expects a 'live' source.
Essentially I think I need to do a variable frame rate to constant frame rate conversion.
My question
How would you normally deal with this issue? How do you do this? Are there ways to do it with a 'live' source in Media Foundation? Or is a manual implementation required (e.g. calculating, dropping frames if faster or duplicating them if slower, etc)?
Code provided below;
#define WIN32_LEAN_AND_MEAN
#include <iostream>
#include <mfapi.h>
#include <d3d11.h>
#include <d3d11_4.h>
#include <dxgi1_5.h>
#include <atlcomcli.h>
#include <mftransform.h>
#include <cassert>
#include <mfidl.h>
#include <mfreadwrite.h>
#include <wmcodecdsp.h>
#include <evr.h>
void SetupEncoder(ID3D11Device*, IMFTransform**);
void SetupFrameRateConverter(ID3D11Device*, IMFTransform**);
void SetupSinkWriter(IMFSinkWriter**);
void InitializeBuffer(IMFTransform*, MFT_OUTPUT_DATA_BUFFER*, UINT32);
int main()
{
SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2);
IDXGIFactory1* dxgiFactory;
auto hr = CreateDXGIFactory1(__uuidof(IDXGIFactory1), (void**)(&dxgiFactory));
IDXGIAdapter* adapter = NULL;
dxgiFactory->EnumAdapters(0, &adapter);
DXGI_ADAPTER_DESC desc = {};
adapter->GetDesc(&desc);
printf("GPU %d: %S (Vendor %04x Device %04x)\n", 0, desc.Description, desc.VendorId, desc.DeviceId);
IDXGIOutput* output;
adapter->EnumOutputs(1, &output);
DXGI_OUTPUT_DESC outputDesc = {};
output->GetDesc(&outputDesc);
printf("Output %S\n", outputDesc.DeviceName);
IDXGIOutput5* dxgiOutput5;
output->QueryInterface(&dxgiOutput5);
// Set up D3D11
D3D_FEATURE_LEVEL featureLevels[] =
{
D3D_FEATURE_LEVEL_11_1,
D3D_FEATURE_LEVEL_11_0,
D3D_FEATURE_LEVEL_10_1,
D3D_FEATURE_LEVEL_10_0,
};
ID3D11Device* device;
D3D_FEATURE_LEVEL levelChosen;
ID3D11DeviceContext* deviceContext;
auto result = D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, D3D11_CREATE_DEVICE_BGRA_SUPPORT, featureLevels, _countof(featureLevels), D3D11_SDK_VERSION, &device, &levelChosen, &deviceContext);
ID3D11Multithread* multithread;
device->QueryInterface(&multithread);
multithread->SetMultithreadProtected(true);
// Set up output duplication
DXGI_FORMAT formats[] =
{
DXGI_FORMAT_B8G8R8A8_UNORM
};
IDXGIOutputDuplication* duplication;
result = dxgiOutput5->DuplicateOutput1(device, 0, _countof(formats), formats, &duplication);
IMFTransform* encoder, * fpsConverter;
IMFSinkWriter* sinkWriter;
SetupEncoder(device, &encoder);
SetupFrameRateConverter(device, &fpsConverter);
SetupSinkWriter(&sinkWriter);
// Allocate buffers
IMFMediaType* outputType;
fpsConverter->GetOutputCurrentType(0, &outputType);
MFT_OUTPUT_DATA_BUFFER buffer;
DWORD status;
UINT32 uiFrameSize = 0;
hr = outputType->GetUINT32(MF_MT_SAMPLE_SIZE, &uiFrameSize);
InitializeBuffer(fpsConverter, &buffer, uiFrameSize);
// Event generator for async MFT
IMFMediaEventGenerator* eventGenerator;
encoder->QueryInterface(&eventGenerator);
// Start up
result = encoder->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, NULL);
result = encoder->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, NULL);
result = encoder->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, NULL);
long startTime = 0;
long frameDuration = (long)((1 / 60.f) * 10000000);
UINT frameCounter = 0;
while (frameCounter < 1000)
{
IMFMediaEvent* mediaEvent;
eventGenerator->GetEvent(0, &mediaEvent);
MediaEventType eventType;
mediaEvent->GetType(&eventType);
if (eventType == METransformNeedInput)
{
// Grab frame first
DXGI_OUTDUPL_FRAME_INFO frameInfo;
IDXGIResource* screenResource;
duplication->AcquireNextFrame(10000, &frameInfo, &screenResource);
ID3D11Texture2D* texture;
screenResource->QueryInterface(&texture);
// Verify correct screen for now
D3D11_TEXTURE2D_DESC d;
texture->GetDesc(&d);
assert(d.Width == 1920);
// Create sample for it
IMFSample* sample;
IMFMediaBuffer* mediaBuffer;
result = MFCreateVideoSampleFromSurface(NULL, &sample);
result = MFCreateDXGISurfaceBuffer(IID_ID3D11Texture2D, texture, 0, TRUE, &mediaBuffer);
result = sample->AddBuffer(mediaBuffer);
////////////////////////
// Does not work, E_UNEXPECTED
// Put it through the FPS converter
/*result = fpsConverter->ProcessInput(0, sample, 0);
if (FAILED(result))
break;
result = fpsConverter->ProcessOutput(0, 1, &buffer, &status);*/
///////////////////////
sample->SetSampleDuration(frameDuration);
sample->SetSampleTime(startTime);
startTime += frameDuration;
result = encoder->ProcessInput(0, sample, 0);
sample->Release();
mediaBuffer->Release();
++frameCounter;
// Important, do not forget to release frame
duplication->ReleaseFrame();
}
else if (eventType == METransformHaveOutput)
{
MFT_OUTPUT_DATA_BUFFER encodingOutputBuffer;
encodingOutputBuffer.dwStreamID = 0;
encodingOutputBuffer.pSample = nullptr;
encodingOutputBuffer.dwStatus = 0;
encodingOutputBuffer.pEvents = nullptr;
result = encoder->ProcessOutput(0, 1, &encodingOutputBuffer, 0);
// Now write to sink
sinkWriter->WriteSample(0, encodingOutputBuffer.pSample);
if (encodingOutputBuffer.pSample)
encodingOutputBuffer.pSample->Release();
if (encodingOutputBuffer.pEvents)
encodingOutputBuffer.pEvents->Release();
}
}
encoder->ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, NULL);
encoder->ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, NULL);
encoder->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, NULL);
result = sinkWriter->Finalize();
sinkWriter->Release();
duplication->Release();
adapter->Release();
device->Release();
}
void SetupEncoder(ID3D11Device* device, IMFTransform** encoderOut)
{
MFStartup(MF_VERSION, MFSTARTUP_FULL);
IMFAttributes* ptr = NULL;
MFCreateAttributes(&ptr, 0);
UINT token;
IMFDXGIDeviceManager* deviceManager;
MFCreateDXGIDeviceManager(&token, &deviceManager);
deviceManager->ResetDevice(device, token);
MFT_REGISTER_TYPE_INFO outputType;
outputType.guidMajorType = MFMediaType_Video;
outputType.guidSubtype = MFVideoFormat_H264;
IMFActivate** activates = NULL;
UINT count = 0;
MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, NULL, &outputType, &activates, &count);
IMFTransform* encoder;
activates[0]->ActivateObject(IID_PPV_ARGS(&encoder));
// Release the rest
for (UINT32 i = 0; i < count; i++)
{
activates[i]->Release();
}
IMFAttributes* attribs;
encoder->GetAttributes(&attribs);
// Required
attribs->SetUINT32(MF_TRANSFORM_ASYNC_UNLOCK, 1);
attribs->SetUINT32(MF_LOW_LATENCY, 1);
LPWSTR friendlyName = 0;
UINT friendlyNameLength;
attribs->GetAllocatedString(MFT_FRIENDLY_NAME_Attribute, &friendlyName, &friendlyNameLength);
printf("Using encoder %S", friendlyName);
auto result = encoder->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, reinterpret_cast<ULONG_PTR>(deviceManager));
DWORD inputStreamId, outputStreamId;
encoder->GetStreamIDs(1, &inputStreamId, 1, &outputStreamId);
// Set up output media type
IMFMediaType* mediaType;
MFCreateMediaType(&mediaType);
mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
mediaType->SetUINT32(MF_MT_AVG_BITRATE, 10240000);
mediaType->SetUINT32(MF_MT_INTERLACE_MODE, 2);
mediaType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, 1);
MFSetAttributeSize(mediaType, MF_MT_FRAME_SIZE, 1920, 1080);
MFSetAttributeRatio(mediaType, MF_MT_FRAME_RATE, 60000, 1001);
result = encoder->SetOutputType(outputStreamId, mediaType, 0);
// Set up input media type
IMFMediaType* suggestedInputType;
result = encoder->GetInputAvailableType(inputStreamId, 0, &suggestedInputType);
suggestedInputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
suggestedInputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12);
MFSetAttributeSize(suggestedInputType, MF_MT_FRAME_SIZE, 1920, 1080);
MFSetAttributeRatio(suggestedInputType, MF_MT_FRAME_RATE, 60000, 1001);
result = encoder->SetInputType(inputStreamId, suggestedInputType, 0);
*encoderOut = encoder;
}
void SetupFrameRateConverter(ID3D11Device* device, IMFTransform** fpsConverterTransformOut)
{
// Set up DSP
IMFTransform* fpsConverter;
CoCreateInstance(CLSID_CFrameRateConvertDmo, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, reinterpret_cast<void**>(&fpsConverter));
// Set up fps input type
IMFMediaType* mediaType;
MFCreateMediaType(&mediaType);
UINT32 imageSize;
MFCalculateImageSize(MFVideoFormat_ARGB32, 1920, 1080, &imageSize);
mediaType->SetUINT32(MF_MT_SAMPLE_SIZE, imageSize);
mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_ARGB32);
MFSetAttributeSize(mediaType, MF_MT_FRAME_SIZE, 1920, 1080);
auto result = fpsConverter->SetInputType(0, mediaType, 0);
// Set up fps output type
MFSetAttributeRatio(mediaType, MF_MT_FRAME_RATE, 60000, 1001);
result = fpsConverter->SetOutputType(0, mediaType, 0);
// Start up FPS
fpsConverter->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, NULL);
fpsConverter->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, NULL);
*fpsConverterTransformOut = fpsConverter;
}
void SetupSinkWriter(IMFSinkWriter** sinkWriterOut)
{
IMFAttributes* attribs;
MFCreateAttributes(&attribs, 0);
attribs->SetUINT32(MF_LOW_LATENCY, 1);
attribs->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1);
attribs->SetGUID(MF_TRANSCODE_CONTAINERTYPE, MFTranscodeContainerType_MPEG4);
attribs->SetUINT32(MF_SINK_WRITER_DISABLE_THROTTLING, 1);
IMFSinkWriter* sinkWriter;
MFCreateSinkWriterFromURL(L"output.mp4", NULL, attribs, &sinkWriter);
// Set up input type
IMFMediaType* mediaType;
MFCreateMediaType(&mediaType);
mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
mediaType->SetUINT32(MF_MT_AVG_BITRATE, 10240000);
MFSetAttributeSize(mediaType, MF_MT_FRAME_SIZE, 1920, 1080);
MFSetAttributeRatio(mediaType, MF_MT_FRAME_RATE, 60000, 1001);
MFSetAttributeRatio(mediaType, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
mediaType->SetUINT32(MF_MT_INTERLACE_MODE, 2);
mediaType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, 0);
DWORD streamIndex;
auto result = sinkWriter->AddStream(mediaType, &streamIndex);
result = sinkWriter->SetInputMediaType(streamIndex, mediaType, NULL);
sinkWriter->BeginWriting();
*sinkWriterOut = sinkWriter;
}
void InitializeBuffer(IMFTransform* transform, MFT_OUTPUT_DATA_BUFFER* buffer, const UINT32 frameSize)
{
MFT_OUTPUT_STREAM_INFO outputStreamInfo;
DWORD outputStreamId = 0;
ZeroMemory(&outputStreamInfo, sizeof(outputStreamInfo));
ZeroMemory(buffer, sizeof(*buffer));
auto hr = transform->GetOutputStreamInfo(outputStreamId, &outputStreamInfo);
if (SUCCEEDED(hr))
{
if ((outputStreamInfo.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES) == 0 &&
(outputStreamInfo.dwFlags & MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES) == 0) {
IMFSample* pOutputSample = NULL;
IMFMediaBuffer* pMediaBuffer = NULL;
hr = MFCreateSample(&pOutputSample);
if (SUCCEEDED(hr)) {
hr = MFCreateMemoryBuffer(frameSize, &pMediaBuffer);
}
if (SUCCEEDED(hr)) {
hr = pOutputSample->AddBuffer(pMediaBuffer);
}
if (SUCCEEDED(hr)) {
buffer->pSample = pOutputSample;
buffer->pSample->AddRef();
}
pMediaBuffer->Release();
pOutputSample->Release();
}
else
{
std::cout << "Stream provides samples";
}
}
}