How to get pixeldata without using LockRect? (Performance issue) - c++

I'm using these function someone post on StackOverflow, but the performance is toooooooo bad..
With out using this function( not capture pixel ) FPS show over 6k, but using this function, FPS show only 60!
So is there any fast method that can copy between video memory and system memory?
Anyone please?
HRESULT capture(IDirect3DDevice9* m_d3ddev, void* buffer, int& width, int& height, D3DFORMAT format)
{
IDirect3DSurface9 *renderTarget = NULL;
IDirect3DSurface9 *destTarget = NULL;
HRESULT hr = m_d3ddev->GetRenderTarget(0, &renderTarget);
hr = m_d3ddev->CreateOffscreenPlainSurface(width, height, format, D3DPOOL_SYSTEMMEM, &destTarget, NULL);
if (FAILED(hr))
{
printf("Failed CreateOffscreenPlainSurface!");
}
hr = m_d3ddev->GetRenderTargetData(renderTarget, destTarget);
if (FAILED(hr))
{
printf("Failed GetRenderTargetData!");
}
D3DLOCKED_RECT lr;
ZeroMemory(&lr, sizeof(D3DLOCKED_RECT));
hr = destTarget->LockRect(&lr, 0, D3DLOCK_READONLY);
if (FAILED(hr))
{
printf("Cannot lock rect!");
}
if (lr.pBits)
{
memcpy(buffer, lr.pBits, width * height * 4);
}
hr = destTarget->UnlockRect();
if (FAILED(hr))
{
printf("Cannot unlock rect!");
}
renderTarget->Release();
destTarget->Release();
return hr;
}

Related

IMFTransform::ProcessInput() "The buffer was too small to carry out the requested action."

I am trying to encode a texture with IMFTransform to H264. I can write and encode textures no problem to a file with SinkWriter and play the video and everything, works great. But I am trying to learn how to use IMFTransform so I can access the encoded IMFSamples themselves.
Unfortunately, I didn't end up getting too far because ProcessInput is failing with "The buffer was too small to carry out the requested action." as the HRESULT.
I have no clue which "buffer" it is referring to, and doing a search of that error turns up absolutely no results. No other calls return a bad HRESULT except ProcessInput(), and SinkWriter works fine. So I have absolutely ZERO clue what the problem is.
#include "main.h"
#include "WinDesktopDup.h"
#include <iostream>
#include <wmcodecdsp.h>
WinDesktopDup dup;
void SetupDpiAwareness()
{
if (!SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_SYSTEM_AWARE))
printf("SetProcessDpiAwarenessContext failed\n");
}
const UINT32 VIDEO_WIDTH = 3840;
const UINT32 VIDEO_HEIGHT = 2160;
const UINT32 VIDEO_FPS = 120;
const UINT64 VIDEO_FRAME_DURATION = 10 * 1000 * 1000 / VIDEO_FPS;
const UINT32 VIDEO_BIT_RATE = 800000;
const GUID VIDEO_ENCODING_FORMAT = MFVideoFormat_H264;
const GUID VIDEO_INPUT_FORMAT = MFVideoFormat_ARGB32;
const UINT32 VIDEO_PELS = VIDEO_WIDTH * VIDEO_HEIGHT;
const UINT32 VIDEO_FRAME_COUNT = 20 * VIDEO_FPS;
template <class T>
void SafeRelease(T** ppT) {
if (*ppT) {
(*ppT)->Release();
*ppT = NULL;
}
}
bool usingEncoder;
IMFMediaType* pMediaTypeOut = NULL;
IMFMediaType* pMediaTypeIn = NULL;
HRESULT SetMediaType()
{
// Set the output media type.
HRESULT hr = MFCreateMediaType(&pMediaTypeOut);
if (!SUCCEEDED(hr)) { printf("MFCreateMediaType failed\n"); }
hr = pMediaTypeOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
if (!SUCCEEDED(hr)) { printf("SetGUID failed\n"); }
hr = pMediaTypeOut->SetGUID(MF_MT_SUBTYPE, VIDEO_ENCODING_FORMAT);
if (!SUCCEEDED(hr)) { printf("SetGUID (2) failed\n"); }
hr = pMediaTypeOut->SetUINT32(MF_MT_AVG_BITRATE, VIDEO_BIT_RATE);
if (!SUCCEEDED(hr)) { printf("SetUINT32 (3) failed\n"); }
hr = pMediaTypeOut->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
if (!SUCCEEDED(hr)) { printf("SetUINT32 (4) failed\n"); }
hr = MFSetAttributeSize(pMediaTypeOut, MF_MT_FRAME_SIZE, VIDEO_WIDTH, VIDEO_HEIGHT);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeSize failed\n"); }
hr = MFSetAttributeRatio(pMediaTypeOut, MF_MT_FRAME_RATE, VIDEO_FPS, 1);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeRatio failed\n"); }
hr = MFSetAttributeRatio(pMediaTypeOut, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeRatio (2) failed\n"); }
// Set the input media type.
hr = MFCreateMediaType(&pMediaTypeIn);
if (!SUCCEEDED(hr)) { printf("MFCreateMediaType failed\n"); }
hr = pMediaTypeIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
if (!SUCCEEDED(hr)) { printf("SetGUID (3) failed\n"); }
hr = pMediaTypeIn->SetGUID(MF_MT_SUBTYPE, VIDEO_INPUT_FORMAT);
if (!SUCCEEDED(hr)) { printf("SetGUID (4) failed\n"); }
hr = pMediaTypeIn->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
if (!SUCCEEDED(hr)) { printf("SetUINT32 (5) failed\n"); }
hr = MFSetAttributeSize(pMediaTypeIn, MF_MT_FRAME_SIZE, VIDEO_WIDTH, VIDEO_HEIGHT);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeSize (2) failed\n"); }
hr = MFSetAttributeRatio(pMediaTypeIn, MF_MT_FRAME_RATE, VIDEO_FPS, 1);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeRatio (3) failed\n"); }
hr = MFSetAttributeRatio(pMediaTypeIn, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
if (!SUCCEEDED(hr)) { printf("MFSetAttributeRatio (4) failed\n"); }
return hr;
}
HRESULT InitializeSinkWriter(IMFSinkWriter** ppWriter, DWORD* pStreamIndex)
{
IMFDXGIDeviceManager* pDeviceManager = NULL;
UINT resetToken;
IMFAttributes* attributes;
*ppWriter = NULL;
*pStreamIndex = NULL;
IMFSinkWriter* pSinkWriter = NULL;
DWORD streamIndex;
HRESULT hr = MFCreateDXGIDeviceManager(&resetToken, &pDeviceManager);
if (!SUCCEEDED(hr)) { printf("MFCreateDXGIDeviceManager failed\n"); }
hr = pDeviceManager->ResetDevice(dup.D3DDevice, resetToken);
if (!SUCCEEDED(hr)) { printf("ResetDevice failed\n"); }
hr = MFCreateAttributes(&attributes, 3);
if (!SUCCEEDED(hr)) { printf("MFCreateAttributes failed\n"); }
hr = attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1);
if (!SUCCEEDED(hr)) { printf("SetUINT32 failed\n"); }
hr = attributes->SetUINT32(MF_LOW_LATENCY, 1);
if (!SUCCEEDED(hr)) { printf("SetUINT32 (2) failed\n"); }
hr = attributes->SetUnknown(MF_SINK_WRITER_D3D_MANAGER, pDeviceManager);
if (!SUCCEEDED(hr)) { printf("SetUnknown failed\n"); }
hr = MFCreateSinkWriterFromURL(L"output.mp4", NULL, attributes, &pSinkWriter);
if (!SUCCEEDED(hr)) { printf("MFCreateSinkWriterFromURL failed\n"); }
hr = pSinkWriter->AddStream(pMediaTypeOut, &streamIndex);
if (!SUCCEEDED(hr)) { printf("AddStream failed\n"); }
hr = pSinkWriter->SetInputMediaType(streamIndex, pMediaTypeIn, NULL);
if (!SUCCEEDED(hr)) { printf("SetInputMediaType failed\n"); }
// Tell the sink writer to start accepting data.
hr = pSinkWriter->BeginWriting();
if (!SUCCEEDED(hr)) { printf("BeginWriting failed\n"); }
// Return the pointer to the caller.
*ppWriter = pSinkWriter;
(*ppWriter)->AddRef();
*pStreamIndex = streamIndex;
SafeRelease(&pSinkWriter);
SafeRelease(&pMediaTypeOut);
SafeRelease(&pMediaTypeIn);
return hr;
}
IUnknown* _transformUnk;
IMFTransform* pMFTransform;
HRESULT InitializeEncoder(DWORD* pStreamIndex)
{
HRESULT hr = CoCreateInstance(CLSID_CMSH264EncoderMFT, NULL, CLSCTX_INPROC_SERVER, IID_IUnknown, (void**)&_transformUnk);
if (!SUCCEEDED(hr)) { printf("CoCreateInstance failed\n"); }
hr = _transformUnk->QueryInterface(IID_PPV_ARGS(&pMFTransform));
if (!SUCCEEDED(hr)) { printf("QueryInterface failed\n"); }
hr = pMFTransform->SetOutputType(0, pMediaTypeOut, 0);
if (!SUCCEEDED(hr)) { printf("SetOutputType failed\n"); }
hr = pMFTransform->SetInputType(0, pMediaTypeIn, 0);
if (!SUCCEEDED(hr)) { printf("SetInputType failed\n"); }
DWORD mftStatus = 0;
hr = pMFTransform->GetInputStatus(0, &mftStatus);
if (!SUCCEEDED(hr)) { printf("GetInputStatus failed\n"); }
if (MFT_INPUT_STATUS_ACCEPT_DATA != mftStatus)
printf("MFT_INPUT_STATUS_ACCEPT_DATA\n");
hr = pMFTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, NULL);
if (!SUCCEEDED(hr)) { printf("MFT_MESSAGE_NOTIFY_BEGIN_STREAMING failed\n"); }
hr = pMFTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, NULL);
if (!SUCCEEDED(hr)) { printf("MFT_MESSAGE_NOTIFY_START_OF_STREAM failed\n"); }
SafeRelease(&pSinkWriter);
SafeRelease(&pMediaTypeOut);
SafeRelease(&pMediaTypeIn);
return hr;
}
ID3D11Texture2D* texture;
HRESULT WriteFrame(IMFSinkWriter* pWriter, DWORD streamIndex, const LONGLONG& rtStart)
{
IMFSample* pSample = NULL;
IMFMediaBuffer* pBuffer = NULL;
HRESULT hr;
hr = MFCreateDXGISurfaceBuffer(__uuidof(ID3D11Texture2D), texture, 0, false, &pBuffer);
if (!SUCCEEDED(hr)) { printf("MFCreateDXGISurfaceBuffer failed\n"); }
DWORD len;
hr = ((IMF2DBuffer*)pBuffer)->GetContiguousLength(&len);
if (!SUCCEEDED(hr)) { printf("GetContiguousLength failed\n"); }
hr = pBuffer->SetCurrentLength(len);
if (!SUCCEEDED(hr)) { printf("SetCurrentLength failed\n"); }
// Create a media sample and add the buffer to the sample.
hr = MFCreateSample(&pSample);
if (!SUCCEEDED(hr)) { printf("MFCreateSample failed\n"); }
hr = pSample->AddBuffer(pBuffer);
if (!SUCCEEDED(hr)) { printf("AddBuffer failed\n"); }
// Set the time stamp and the duration.
hr = pSample->SetSampleTime(rtStart);
if (!SUCCEEDED(hr)) { printf("SetSampleTime failed\n"); }
hr = pSample->SetSampleDuration(VIDEO_FRAME_DURATION);
if (!SUCCEEDED(hr)) { printf("SetSampleDuration failed\n"); }
// Send the sample to the Sink Writer or Encoder.
if (!usingEncoder)
{
hr = pWriter->WriteSample(streamIndex, pSample);
if (!SUCCEEDED(hr)) { printf("WriteSample failed\n"); }
}
else
{
hr = pMFTransform->ProcessInput(0, pSample, 0);
if (!SUCCEEDED(hr)) { printf("ProcessInput failed\n"); }
}
SafeRelease(&pSample);
SafeRelease(&pBuffer);
return hr;
}
int APIENTRY main(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nShowCmd)
{
SetupDpiAwareness();
auto err = dup.Initialize();
// Initialize MF
CoInitializeEx(0, COINIT_APARTMENTTHREADED); // Need to call this once when a thread is using COM or it wont work
MFStartup(MF_VERSION); // Need to call this too for Media Foundation related memes
IMFSinkWriter* pSinkWriter = NULL;
DWORD stream = 0;
LONGLONG rtStart = 0;
usingEncoder = true; // True if we want to encode with IMFTransform, false if we want to write with SinkWriter
HRESULT hr = SetMediaType();
if (!SUCCEEDED(hr)) { printf("SetMediaType failed\n"); }
if (!usingEncoder)
{
hr = InitializeSinkWriter(&pSinkWriter, &stream);
if (!SUCCEEDED(hr)) { printf("InitializeSinkWriter failed\n"); }
}
else
{
hr = pMediaTypeIn->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_IYUV); // Using MFVideoFormat_ARGB32 causes SetInputType() to fail
hr = InitializeEncoder(&stream);
if (!SUCCEEDED(hr)) { printf("InitializeEncoder failed\n"); }
}
const int CAPTURE_LENGTH = 10;
int total_frames = VIDEO_FPS * CAPTURE_LENGTH;
for (int i = 0; i < 1; i++)
{
texture = dup.CaptureNext();
if (texture != nullptr)
{
hr = WriteFrame(pSinkWriter, stream, rtStart);
if (!SUCCEEDED(hr))
printf("WriteFrame failed\n");
rtStart += VIDEO_FRAME_DURATION;
texture->Release();
}
else
{
i--;
}
}
if (FAILED(hr))
{
std::cout << "Failure" << std::endl;
}
if (SUCCEEDED(hr)) {
hr = pSinkWriter->Finalize();
}
SafeRelease(&pSinkWriter);
MFShutdown();
CoUninitialize();
}
Here’s documentation for the Microsoft’s software CPU-based h.264 encoder you’re using in your code.
It does not support MFVideoFormat_ARGB32 on input. It doesn’t support any RGB formats at all. That transform only supports YUV formats for the input video.
BTW, if you replace the MFT with a hardware encoder, they’re very likely to expose same set of features as the Microsoft’s software one, I don’t think they support RGB. And, because all hardware transforms are asynchronous, you gonna need slightly different workflow to drive them directly.
The reason sink writer works OK, it creates and hosts 2 MFTs under the hood, the format converter from RGB to YUV, another one is the encoder.
You have following options.
Use another MFT to convert RGBA to NV12 before passing frames to the encoder.
Do that conversation yourself with pixel shaders (render a textured quad into 2 planes of NV12 texture using 2 different pixel shaders), or with a single compute shader (dispatch 1 thread for every 2x2 block of the video, write 6 bytes for every block, 4 into R8_UNORM output texture with brightness, other 2 bytes into R8G8_UNORM output texture with color data).
Use a sink writer, but create it with MFCreateSinkWriterFromMediaSink API instead of MFCreateSinkWriterFromURL. Implement IMFMediaSink COM interface, also IMFStreamSink for it’s video stream, and the framework will call IMFStreamSink.ProcessSample giving you encoded video samples in system memory as soon as they’re available.

Direct2D Loading and Drawing Bitmap

I'm trying to draw a bitmap to the screen; the code that calls to load the bitmap from a file works, but it seems the actual drawing part is causing some program crashes.
I followed a tutorial from MSDN, but the only change I made was instead of using ID2D1RenderTarget, I'm using a ID2D1DCRenderTarget.
Here's the Load method that works:
HRESULT LoadBitmapFromFile(
ID2D1DCRenderTarget *pRenderTarget,
IWICImagingFactory *pIWICFactory,
PCWSTR uri,
UINT destinationWidth,
UINT destinationHeight,
ID2D1Bitmap **ppBitmap
)
{
IWICBitmapDecoder *pDecoder = NULL;
IWICBitmapFrameDecode *pSource = NULL;
IWICStream *pStream = NULL;
IWICFormatConverter *pConverter = NULL;
IWICBitmapScaler *pScaler = NULL;
HRESULT hr = pIWICFactory->CreateDecoderFromFilename(
uri,
NULL,
GENERIC_READ,
WICDecodeMetadataCacheOnLoad,
&pDecoder
);
if (SUCCEEDED(hr))
{
// Create the initial frame.
hr = pDecoder->GetFrame(0, &pSource);
}
if (SUCCEEDED(hr))
{
// Convert the image format to 32bppPBGRA
// (DXGI_FORMAT_B8G8R8A8_UNORM + D2D1_ALPHA_MODE_PREMULTIPLIED).
hr = pIWICFactory->CreateFormatConverter(&pConverter);
}
if (SUCCEEDED(hr))
{
// If a new width or height was specified, create an
// IWICBitmapScaler and use it to resize the image.
if (destinationWidth != 0 || destinationHeight != 0)
{
UINT originalWidth, originalHeight;
hr = pSource->GetSize(&originalWidth, &originalHeight);
if (SUCCEEDED(hr))
{
if (destinationWidth == 0)
{
FLOAT scalar = static_cast<FLOAT>(destinationHeight) / static_cast<FLOAT>(originalHeight);
destinationWidth = static_cast<UINT>(scalar * static_cast<FLOAT>(originalWidth));
}
else if (destinationHeight == 0)
{
FLOAT scalar = static_cast<FLOAT>(destinationWidth) / static_cast<FLOAT>(originalWidth);
destinationHeight = static_cast<UINT>(scalar * static_cast<FLOAT>(originalHeight));
}
hr = pIWICFactory->CreateBitmapScaler(&pScaler);
if (SUCCEEDED(hr))
{
hr = pScaler->Initialize(
pSource,
destinationWidth,
destinationHeight,
WICBitmapInterpolationModeCubic
);
}
if (SUCCEEDED(hr))
{
hr = pConverter->Initialize(
pScaler,
GUID_WICPixelFormat32bppPBGRA,
WICBitmapDitherTypeNone,
NULL,
0.f,
WICBitmapPaletteTypeMedianCut
);
}
}
}
else // Don't scale the image.
{
hr = pConverter->Initialize(
pSource,
GUID_WICPixelFormat32bppPBGRA,
WICBitmapDitherTypeNone,
NULL,
0.f,
WICBitmapPaletteTypeMedianCut
);
}
}
if (SUCCEEDED(hr))
{
// Create a Direct2D bitmap from the WIC bitmap.
hr = pRenderTarget->CreateBitmapFromWicBitmap(
pConverter,
NULL,
ppBitmap
);
}
if (pDecoder) pDecoder->Release();
if (pSource) pSource->Release();
if (pStream) pStream->Release();
if (pScaler) pScaler->Release();
return hr;
}
And in between the BeginDraw() and EndDraw() for the render target, I have the following code to draw the bitmap. This is the part that crashes the program when I attempt to run it:
d2dg->pRT->BeginDraw();
// Create a WIC Factory
HRESULT hr = CoCreateInstance(
CLSID_WICImagingFactory,
NULL,
CLSCTX_INPROC_SERVER,
IID_IWICImagingFactory,
(LPVOID*)&wicFactory);
hr = LoadBitmapFromFile(d2dg->pRT, wicFactory, L"image1.png", 400, 400, &pBitmap);
D2D1_SIZE_F size = pBitmap->GetSize();
D2D1_POINT_2F upperLeftCorner = D2D1::Point2F(100.f, 10.f);
// Draw a bitmap.
d2dg->pRT->DrawBitmap(
pBitmap,
D2D1::RectF(
upperLeftCorner.x,
upperLeftCorner.y,
upperLeftCorner.x + size.width,
upperLeftCorner.y + size.height)
);
d2dg->pRT->EndDraw();
Found the issue to this: the bitmap I wanted to draw was still NULL.
This can be fixed by deleting the ID2D1Bitmap **ppBitmap parameter from LoadBitmapFromFile and just work with the pointer to &pBitmap for any references to a bitmap within the method.
That way, pBitmap gets manipulated and will not remain NULL.

Video captured by Media Foundation is vertical mirrorred

I'm using Media Foundation IMFSourceReaderCallback implementation for grabbing video frames from the camera, and then OpenCV imshow to present the frames in a loop.
However I get the frames vertically flipped...
Is this a bug? Should I set some attribute to avoid this?
Here is my code:
Initialization:
IMFAttributes* pDeviceAttrs, *pReaderAttrs;
hr = MFCreateAttributes(&pDeviceAttrs, 1);
if (FAILED(hr)) goto Exit;
hr = pDeviceAttrs->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID);
if (FAILED(hr)) goto Exit;
//...
// Correct source provider is activated through ActivateObject
//
hr = MFCreateAttributes(&pReaderAttrs, 2);
if (FAILED(hr)) goto Exit;
pReaderAttrs->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK,(IUnknown*)this);
pReaderAttrs->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE);
hr = MFCreateSourceReaderFromMediaSource(pMediaSource, pReaderAttrs, &m_pReader);
if (FAILED(hr)) goto Exit;
// Correct profile is set
OnReadSample implementation:
HRESULT hr = S_OK;
LONG defaultStride = 0;
LONG stride = 0;
BYTE *pBuffer = NULL;
EnterCriticalSection(&m_critSec);
if (NULL != pSample)
{
IMFMediaBuffer* pMediaBuffer;
DWORD dataSize = 0;
// In case of a single buffer, no copy would happen
hr = pSample->ConvertToContiguousBuffer(&pMediaBuffer);
if (FAILED(hr)) goto Cleanup;
pMediaBuffer->GetCurrentLength(&dataSize);
hr = pMediaBuffer->Lock(&pBuffer, &dataSize, &dataSize);
if (FAILED(hr)) goto Cleanup;
// todo: use a backbuffer to avoid sync issues
if (NULL == m_pLatestFrame) m_pLatestFrame = (BYTE*)malloc(dataSize);
memcpy(m_pLatestFrame, pBuffer, dataSize);
++m_frameNumber;
pMediaBuffer->Unlock();
pMediaBuffer->Release();
}
Cleanup:
LeaveCriticalSection(&m_critSec);
// Async ReadFrame for the next buffer:
hr = m_pReader->ReadSample(
(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
0,
NULL, // actual
NULL, // flags
NULL, // timestamp
NULL // sample
);
return hr;
Conversion to cv::image:
void SourceReaderImpl::GetLatestFrame(BYTE** ppLatestFrame)
{
EnterCriticalSection(&m_critSec);
*ppLatestFrame = m_pLatestFrame;
LeaveCriticalSection(&m_critSec);
}
void* CameraWrapperImpl::getLatestFrame()
{
BYTE* pLatestFrame = NULL;
m_pMfReader->GetLatestFrame(&pLatestFrame);
return pLatestFrame;
}
void Player::Present()
{
//...
color = cv::Mat(colorSize,
CV_8UC3,
static_cast<unsigned char*>(m_pColorCameraImpl->getLatestFrame()));
cv::imshow(color);
}
Any idea?
Thanks in advance!
A bitmap is stored with the last scan line first, so the image will appear upside down. The easiest solution is to call cv::flip
void Player::Present()
{
//...
color = cv::Mat(colorSize,
CV_8UC3,
static_cast<unsigned char*>(m_pColorCameraImpl->getLatestFrame()));
cv::Mat corrected;
flip(color, corrected, 0);
imshow(corrected);
}

ISampleGrabber::GetCurrentBuffer() returns E_OUTOFMEMORY

I am new to directshow and using DirectShow Sample "FrameGrabberDemo" and facing issue in getting the image. I tried with .avi and .mpg, both are giving same issue.
First issue might be the S_FALSE return value from IMediaControl::Run(). However, it is not an error and states that:
The graph is preparing to run, but some filters have not completed the
transition to a running state.
Second observation is ISampleGrabber::GetCurrentBuffer() returns E_OUTOFMEMORY code, which states that "The specified buffer is not large enough." However, the BitmapInfo has biImageSize = 1244160 and also MediaType has ISampleSize = 1244160.
HRESULT CFrameGrabberDemoDlg::DoExtractFrame()
{
WCHAR wFile[MAX_PATH];
MultiByteToWideChar( CP_ACP, 0, m_FilePath, -1, wFile, MAX_PATH );
// Create the graph builder
CComPtr<IGraphBuilder> pGraphBuilder;
HRESULT hr = ::CoCreateInstance(CLSID_FilterGraph, NULL, CLSCTX_INPROC_SERVER,
IID_IGraphBuilder, (void**)&pGraphBuilder);
if (FAILED(hr))
return hr;
ASSERT(pGraphBuilder != NULL);
// Create the "Grabber filter"
CComPtr<IBaseFilter> pGrabberBaseFilter;
CComPtr<ISampleGrabber> pSampleGrabber;
AM_MEDIA_TYPE mt;
hr = ::CoCreateInstance(CLSID_SampleGrabber, NULL, CLSCTX_INPROC_SERVER,
IID_IBaseFilter, (LPVOID *)&pGrabberBaseFilter);
if (FAILED(hr))
return hr;
pGrabberBaseFilter->QueryInterface(IID_ISampleGrabber, (void**)&pSampleGrabber);
if (pSampleGrabber == NULL)
return E_NOINTERFACE;
hr = pGraphBuilder->AddFilter(pGrabberBaseFilter,L"Grabber");
if (FAILED(hr))
return hr;
ZeroMemory(&mt, sizeof(AM_MEDIA_TYPE));
mt.majortype = MEDIATYPE_Video;
mt.subtype = MEDIASUBTYPE_RGB24;
mt.formattype = FORMAT_VideoInfo;
hr = pSampleGrabber->SetMediaType(&mt);
if (FAILED(hr))
return hr;
hr = pGraphBuilder->RenderFile(wFile,NULL);
if (FAILED(hr))
return hr;
CComPtr<IMediaControl> pMediaControl;
CComPtr<IMediaEvent> pMediaEventEx;
// QueryInterface for some basic interfaces
pGraphBuilder->QueryInterface(IID_IMediaControl, (void **)&pMediaControl);
pGraphBuilder->QueryInterface(IID_IMediaEvent, (void **)&pMediaEventEx);
if (pMediaControl == NULL || pMediaEventEx == NULL)
return E_NOINTERFACE;
// Set up one-shot mode.
hr = pSampleGrabber->SetBufferSamples(TRUE);
if (FAILED(hr))
return hr;
hr = pSampleGrabber->SetOneShot(TRUE);
if (FAILED(hr))
return hr;
CComQIPtr<IMediaSeeking> pSeek = pMediaControl;
if (pSeek == NULL)
return E_NOINTERFACE;
LONGLONG Duration;
hr = pSeek->GetDuration(&Duration);
if (FAILED(hr))
return hr;
int NumSecs = int(Duration/10000000);
REFERENCE_TIME rtStart = 1 * 10000000;
if (NumSecs < 1)
rtStart = 0;
REFERENCE_TIME rtStop = rtStart;
hr = pSeek->SetPositions(&rtStart, AM_SEEKING_AbsolutePositioning,
&rtStop, AM_SEEKING_AbsolutePositioning);
if (FAILED(hr))
return hr;
CComQIPtr<IVideoWindow> pVideoWindow = pGraphBuilder;
hr = pVideoWindow->put_AutoShow(OAFALSE);
if (FAILED(hr))
return hr;
// Run the graph and wait for completion.
hr = pMediaControl->Run();
if (FAILED(hr))
return hr;
long evCode;
hr = pMediaEventEx->WaitForCompletion(INFINITE, &evCode);
if (FAILED(hr))
return hr;
AM_MEDIA_TYPE MediaType;
ZeroMemory(&MediaType,sizeof(MediaType));
hr = pSampleGrabber->GetConnectedMediaType(&MediaType);
if (FAILED(hr))
return hr;
// Get a pointer to the video header.
VIDEOINFOHEADER *pVideoHeader = (VIDEOINFOHEADER*)MediaType.pbFormat;
if (pVideoHeader == NULL)
return E_FAIL;
// The video header contains the bitmap information.
// Copy it into a BITMAPINFO structure.
BITMAPINFO BitmapInfo;
ZeroMemory(&BitmapInfo, sizeof(BitmapInfo));
CopyMemory(&BitmapInfo.bmiHeader, &(pVideoHeader->bmiHeader), sizeof(BITMAPINFOHEADER));
// Create a DIB from the bitmap header, and get a pointer to the buffer.
void *buffer = NULL;
HBITMAP hBitmap = ::CreateDIBSection(0, &BitmapInfo, DIB_RGB_COLORS, &buffer, NULL, 0);
GdiFlush();
// Copy the image into the buffer.
long size = 0;
hr = pSampleGrabber->GetCurrentBuffer(&size, (long *)buffer);
if (FAILED(hr))
return hr;
long Width = pVideoHeader->bmiHeader.biWidth;
long Height = pVideoHeader->bmiHeader.biHeight;
HBITMAP hOldBitmap = m_Image.SetBitmap(hBitmap);
if (hOldBitmap != NULL)
::DeleteObject(hOldBitmap);
return S_OK;
}
The ::CreateDIBSection is also not returning NULL and buffer also got initialized.
How can this be resolved?
You are requesting data into zero-length buffer:
long size = 0;
hr = pSampleGrabber->GetCurrentBuffer(&size, (long *)buffer);
The error code looks relevant:
If pBuffer is not NULL, set this parameter equal to the size of the buffer, in bytes.
E_OUTOFMEMORY The specified buffer is not large enough.
You simple need proper arguments in the call in question (proper buffer length).

DirectShow: webcam preview and image capture

After looking at a very similar question and seeing almost identical code, I've decided to ask this question separately. I want to show a video preview of the webcam's video stream to the default window DirectShow uses, and I also want the ability to "take a picture" of the video stream at any given moment.
I started with the DirectShow examples on MSDN, as well as the AMCap sample code, and have something I believe should should the preview part, but does not. I've found no examples of grabbing an image from the video stream except using SampleGrabber, which is deprecated and therefore I am trying not to use it.
Below is my code, line for line. Note that most of the code in EnumerateCameras is commented out. That code would've been for attaching to another window, which I don't want to do. In the MSDN documentation, it explicitly states that the VMR_7 creates its own window to display the video stream. I get no errors in my app, but this window never appears.
My question then is this: What am I doing wrong? Alternatively, if you know of a simple example of what I am trying to do, link me to it. AMCap is not a simple example, for reference.
NOTE: InitalizeVMR is for running in windowless state, which is my ultimate goal (integrating into a DirectX game). For now, however, i just want it to run in the simplest mode possible.
EDIT: The first portion of this question, that is previewing the camera stream, is solved. I am now just looking for an alternative to the deprecated SampleGrabber class so I can snap a photo at any moment and save it to a file.
EDIT: After looking for almost an hour on google, the general concensus seems to be that you HAVE to use ISampleGrabber. Please let me know if you find anything different.
Testing code (main.cpp):
CWebcam* camera = new CWebcam();
HRESULT hr = CoInitializeEx(NULL, COINIT_MULTITHREADED);
MessageBox(NULL, L"text", L"caption", NULL);
if (SUCCEEDED(hr))
{
camera->Create();
camera->EnumerateCameras();
camera->StartCamera();
}
int d;
cin >> d;
Webcam.cpp:
#include "Webcam.h"
CWebcam::CWebcam() {
HRESULT hr = CoInitializeEx(NULL, COINIT_MULTITHREADED);
//m_pTexInst = nullptr;
//m_pTexRes = nullptr;
}
CWebcam::~CWebcam() {
CoUninitialize();
m_pDeviceMonikers->Release();
m_pMediaController->Release();
}
BOOL CWebcam::Create() {
InitCaptureGraphBuilder(&m_pFilterGraph, &m_pCaptureGraph);
hr = m_pFilterGraph->QueryInterface(IID_IMediaControl, (void **)&m_pMediaController);
return TRUE;
}
void CWebcam::Destroy() {
}
void CWebcam::EnumerateCameras() {
HRESULT hr = EnumerateDevices(CLSID_VideoInputDeviceCategory, &m_pDeviceMonikers);
if (SUCCEEDED(hr))
{
//DisplayDeviceInformation(m_pDeviceMonikers);
//m_pDeviceMonikers->Release();
IMoniker *pMoniker = NULL;
if(m_pDeviceMonikers->Next(1, &pMoniker, NULL) == S_OK)
{
hr = pMoniker->BindToObject(0, 0, IID_IBaseFilter, (void**)&m_pCameraFilter);
if (SUCCEEDED(hr))
{
hr = m_pFilterGraph->AddFilter(m_pCameraFilter, L"Capture Filter");
}
}
// connect the output pin to the video renderer
if(SUCCEEDED(hr))
{
hr = m_pCaptureGraph->RenderStream(&PIN_CATEGORY_PREVIEW, &MEDIATYPE_Video,
m_pCameraFilter, NULL, NULL);
}
//InitializeVMR(hwnd, m_pFilterGraph, &m_pVMRControl, 1, FALSE);
//get the video window that will be displayed from the filter graph
IVideoWindow *pVideoWindow = NULL;
hr = m_pFilterGraph->QueryInterface(IID_IVideoWindow, (void **)&pVideoWindow);
/*if(hr != NOERROR)
{
printf("This graph cannot preview properly");
}
else
{
//get the video stream configurations
hr = m_pCaptureGraph->FindInterface(&PIN_CATEGORY_CAPTURE,
&MEDIATYPE_Video, m_pCameraFilter,
IID_IAMStreamConfig, (void **)&m_pVideoStreamConfig);
//Find out if this is a DV stream
AM_MEDIA_TYPE *pMediaTypeDV;
//fake window handle
HWND window = NULL;
if(m_pVideoStreamConfig && SUCCEEDED(m_pVideoStreamConfig->GetFormat(&pMediaTypeDV)))
{
if(pMediaTypeDV->formattype == FORMAT_DvInfo)
{
// in this case we want to set the size of the parent window to that of
// current DV resolution.
// We get that resolution from the IVideoWindow.
IBasicVideo* pBasivVideo;
// If we got here, gcap.pVW is not NULL
//ASSERT(pVideoWindow != NULL);
hr = pVideoWindow->QueryInterface(IID_IBasicVideo, (void**)&pBasivVideo);
/*if(SUCCEEDED(hr))
{
HRESULT hr1, hr2;
long lWidth, lHeight;
hr1 = pBasivVideo->get_VideoHeight(&lHeight);
hr2 = pBasivVideo->get_VideoWidth(&lWidth);
if(SUCCEEDED(hr1) && SUCCEEDED(hr2))
{
ResizeWindow(lWidth, abs(lHeight));
}
}
}
}
RECT rc;
pVideoWindow->put_Owner((OAHWND)window); // We own the window now
pVideoWindow->put_WindowStyle(WS_CHILD); // you are now a child
GetClientRect(window, &rc);
pVideoWindow->SetWindowPosition(0, 0, rc.right, rc.bottom); // be this big
pVideoWindow->put_Visible(OATRUE);
}*/
}
}
BOOL CWebcam::StartCamera() {
if(m_bIsStreaming == FALSE)
{
m_bIsStreaming = TRUE;
hr = m_pMediaController->Run();
if(FAILED(hr))
{
// stop parts that ran
m_pMediaController->Stop();
return FALSE;
}
return TRUE;
}
return FALSE;
}
void CWebcam::EndCamera() {
if(m_bIsStreaming)
{
hr = m_pMediaController->Stop();
m_bIsStreaming = FALSE;
//invalidate client rect as well so that it must redraw
}
}
BOOL CWebcam::CaptureToTexture() {
return TRUE;
}
HRESULT CWebcam::InitCaptureGraphBuilder(
IGraphBuilder **ppGraph, // Receives the pointer.
ICaptureGraphBuilder2 **ppBuild // Receives the pointer.
)
{
if (!ppGraph || !ppBuild)
{
return E_POINTER;
}
IGraphBuilder *pGraph = NULL;
ICaptureGraphBuilder2 *pBuild = NULL;
// Create the Capture Graph Builder.
HRESULT hr = CoCreateInstance(CLSID_CaptureGraphBuilder2, NULL,
CLSCTX_INPROC_SERVER, IID_ICaptureGraphBuilder2, (void**)&pBuild );
if (SUCCEEDED(hr))
{
// Create the Filter Graph Manager.
hr = CoCreateInstance(CLSID_FilterGraph, 0, CLSCTX_INPROC_SERVER,
IID_IGraphBuilder, (void**)&pGraph);
if (SUCCEEDED(hr))
{
// Initialize the Capture Graph Builder.
pBuild->SetFiltergraph(pGraph);
// Return both interface pointers to the caller.
*ppBuild = pBuild;
*ppGraph = pGraph; // The caller must release both interfaces.
return S_OK;
}
else
{
pBuild->Release();
}
}
return hr; // Failed
}
HRESULT CWebcam::EnumerateDevices(REFGUID category, IEnumMoniker **ppEnum)
{
// Create the System Device Enumerator.
ICreateDevEnum *pSystemDeviceEnumerator;
HRESULT hr = CoCreateInstance(CLSID_SystemDeviceEnum, NULL,
CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&pSystemDeviceEnumerator));
if (SUCCEEDED(hr))
{
// Create an enumerator for the category.
hr = pSystemDeviceEnumerator->CreateClassEnumerator(category, ppEnum, 0);
if (hr == S_FALSE)
{
hr = VFW_E_NOT_FOUND; // The category is empty. Treat as an error.
}
pSystemDeviceEnumerator->Release();
}
return hr;
}
void CWebcam::DisplayDeviceInformation(IEnumMoniker *pEnum)
{
IMoniker *pMoniker = NULL;
int counter = 0;
while (pEnum->Next(1, &pMoniker, NULL) == S_OK)
{
IPropertyBag *pPropBag;
HRESULT hr = pMoniker->BindToStorage(0, 0, IID_PPV_ARGS(&pPropBag));
if (FAILED(hr))
{
pMoniker->Release();
continue;
}
VARIANT var;
VariantInit(&var);
// Get description or friendly name.
hr = pPropBag->Read(L"Description", &var, 0);
if (FAILED(hr))
{
hr = pPropBag->Read(L"FriendlyName", &var, 0);
}
if (SUCCEEDED(hr))
{
printf("%d: %S\n", counter, var.bstrVal);
VariantClear(&var);
}
hr = pPropBag->Write(L"FriendlyName", &var);
// WaveInID applies only to audio capture devices.
hr = pPropBag->Read(L"WaveInID", &var, 0);
if (SUCCEEDED(hr))
{
printf("%d: WaveIn ID: %d\n", counter, var.lVal);
VariantClear(&var);
}
hr = pPropBag->Read(L"DevicePath", &var, 0);
if (SUCCEEDED(hr))
{
// The device path is not intended for display.
printf("%d: Device path: %S\n", counter, var.bstrVal);
VariantClear(&var);
}
pPropBag->Release();
pMoniker->Release();
counter++;
}
}
HRESULT CWebcam::InitializeVMR(
HWND hwndApp, // Application window.
IGraphBuilder* pFG, // Pointer to the Filter Graph Manager.
IVMRWindowlessControl** ppWc, // Receives the interface.
DWORD dwNumStreams, // Number of streams to use.
BOOL fBlendAppImage // Are we alpha-blending a bitmap?
)
{
IBaseFilter* pVmr = NULL;
IVMRWindowlessControl* pWc = NULL;
*ppWc = NULL;
// Create the VMR and add it to the filter graph.
HRESULT hr = CoCreateInstance(CLSID_VideoMixingRenderer, NULL,
CLSCTX_INPROC, IID_IBaseFilter, (void**)&pVmr);
if (FAILED(hr))
{
return hr;
}
hr = pFG->AddFilter(pVmr, L"Video Mixing Renderer");
if (FAILED(hr))
{
pVmr->Release();
return hr;
}
// Set the rendering mode and number of streams.
IVMRFilterConfig* pConfig;
hr = pVmr->QueryInterface(IID_IVMRFilterConfig, (void**)&pConfig);
if (SUCCEEDED(hr))
{
pConfig->SetRenderingMode(VMRMode_Windowless);
// Set the VMR-7 to mixing mode if you want more than one video
// stream, or you want to mix a static bitmap over the video.
// (The VMR-9 defaults to mixing mode with four inputs.)
if (dwNumStreams > 1 || fBlendAppImage)
{
pConfig->SetNumberOfStreams(dwNumStreams);
}
pConfig->Release();
hr = pVmr->QueryInterface(IID_IVMRWindowlessControl, (void**)&pWc);
if (SUCCEEDED(hr))
{
pWc->SetVideoClippingWindow(hwndApp);
*ppWc = pWc; // The caller must release this interface.
}
}
pVmr->Release();
// Now the VMR can be connected to other filters.
return hr;
}
In windowless mode VMR would not create separate window. Since you started initialization for widnowless mode, you have to follow SetVideoClippingWindow with IVMRWindowlessControl::SetVideoPosition call to provide position within the window, see VMR Windowless Mode on MSDN.
Another sample code snippet for you: http://www.assembla.com/code/roatl-utilities/subversion/nodes/trunk/FullScreenWindowlessVmrSample01/MainDialog.h#ln188