MediaFoundation + VP8 + Color Formats - c++

Question

My ultimate goal is to add video support to my Virtual Class application. My method is:
Capture frames using Media Foundation
Encode with VP8 using LibVPX
Transmit with UDP
Decode at the receipient site
Show frames in a Window
The first problem arises with my WebCam's supported color encodings. The webcam Media Types contain only MFVideoFormat_NV12. My first debugging attempt saves the receiving image to a bitmap so I can test it is correctly captured (error handling removed):
HRESULT CAP::StartRecord(HWND hh, CComPtr<IMFMediaSource> src)
{
MFCreateSourceReaderFromMediaSource(src, 0, &sr);
CComPtr<IMFMediaType> fmt;
sr->GetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM,&fmt);
LogMediaType(fmt); // Shows: MFVideoFormat_NV12
auto [wi, he] = WidthHeight(fmt);
for (;;)
{
DWORD streamIndex = 0, flags = 0;
LONGLONG llTimeStamp = 0;
CComPtr<IMFSample> pSample;
hr = sr->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM,0,&streamIndex,&flags,&llTimeStamp,&pSample);
if (FAILED(hr))
break;
if (!pSample)
continue;
CComPtr<IMFMediaBuffer> bu;
pSample->ConvertToContiguousBuffer(&bu);
SaveSampleNV12(bu, wi, he);
}
...
}
SaveSampleNV12 uses code from here to convert NV12 to RGB, then:
void SaveSampleNV12(CComPtr<IMFMediaBuffer> mm, int width32, int height32)
{
DWORD le = 0;
mm->GetCurrentLength(&le);
BYTE *pDatad = NULL;
auto hr = mm->Lock(&pDatad, NULL, NULL);
vector<char> rgb(1000000);
NV12ToRGB((BYTE*)rgb.data(), pDatad, width32, height32);
mm->Unlock();
HANDLE file;
BITMAPFILEHEADER fileHeader;
BITMAPINFOHEADER fileInfo;
DWORD write = 0;
auto df = L"r:\\f.bmp";
file = CreateFile(df.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); //Sets up the new bmp to be written to
int bits = 24;
fileHeader.bfType = 19778; //Sets our type to BM or bmp
fileHeader.bfSize = sizeof(fileHeader.bfOffBits) + sizeof(RGBTRIPLE); //Sets the size equal to the size of the header struct
fileHeader.bfReserved1 = 0; //sets the reserves to 0
fileHeader.bfReserved2 = 0;
fileHeader.bfOffBits = sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER); //Sets offbits equal to the size of file and info header
fileInfo.biSize = sizeof(BITMAPINFOHEADER);
fileInfo.biWidth = width32;
fileInfo.biHeight = height32;
fileInfo.biPlanes = 1;
fileInfo.biBitCount = bits;
fileInfo.biCompression = BI_RGB;
fileInfo.biSizeImage = width32 * height32 * (bits / 8);
fileInfo.biXPelsPerMeter = 0;// 2400;
fileInfo.biYPelsPerMeter = 0;// 2400;
fileInfo.biClrImportant = 0;
fileInfo.biClrUsed = 0;
WriteFile(file, &fileHeader, sizeof(fileHeader), &write, NULL);
WriteFile(file, &fileInfo, sizeof(fileInfo), &write, NULL);
unsigned char* ptrIn = (unsigned char*)rgb.data();
int rgbs = width32 * height32 * (bits / 8);
vector<char> d2(rgbs);
unsigned char* ptrOut = (unsigned char*)d2.data();
for (int i = 0; i < (width32*height32) / 2; ++i)
{
int y0 = ptrIn[0];
int u0 = ptrIn[1];
int y1 = ptrIn[2];
int v0 = ptrIn[3];
ptrIn += 4;
int c = y0 - 16;
int d = u0 - 128;
int e = v0 - 128;
int bb = clip((298 * c + 516 * d + 128) >> 8); // blue
int gg = clip((298 * c - 100 * d - 208 * e + 128) >> 8); // green
int rr = clip((298 * c + 409 * e + 128) >> 8); // red
ptrOut[0] = bb;
ptrOut[1] = gg;
ptrOut[2] = rr;
c = y1 - 16;
ptrOut[3] = clip((298 * c + 516 * d + 128) >> 8); // blue
ptrOut[4] = clip((298 * c - 100 * d - 208 * e + 128) >> 8); // green
ptrOut[5] = clip((298 * c + 409 * e + 128) >> 8); // red
ptrOut += 6;
}
unsigned char* cc = (unsigned char*)d2.data();
WriteFile(file, cc, rgbs, &write, NULL);
CloseHandle(file);
}
This returns a weird full of Pink image. Something I'm doing wrong, but what?
Thanks a lot.

Answer 1

The solution is to use IMFTransform to tranform between various color spaces.