I'm working on a WASAPI UWP audio application with cpp/winrt which needs to take audio from an input and send it to an output after being processed.
I want to set my audio thread characteristics with AvSetMmThreadCharacteristicsW(L"Pro Audio", &taskIndex), but I just noticed this function (and most of avrt.h) is limited to WINAPI_PARTITION_DESKTOP and WINAPI_PARTITION_GAMES.
I think I need this because when my code is integrated into my UWP app, the audio input is full of discontinuity, and I have no issue in my test code which uses the avrt API.
Is there another way to configure my thread for audio processing?
Edit: here is my test program https://github.com/loics2/test-wasapi. The interesting part happens in the AudioStream class. I can't share my UWP app, but I can copy as is these classes into a Windows Runtime Component.
Edit 2: here's the audio thread code :
void AudioStream::StreamWorker()
{
WAVEFORMATEX* captureFormat = nullptr;
WAVEFORMATEX* renderFormat = nullptr;
RingBuffer<float> captureBuffer;
RingBuffer<float> renderBuffer;
BYTE* streamBuffer = nullptr;
unsigned int streamBufferSize = 0;
unsigned int bufferFrameCount = 0;
unsigned int numFramesPadding = 0;
unsigned int inputBufferSize = 0;
unsigned int outputBufferSize = 0;
DWORD captureFlags = 0;
winrt::hresult hr = S_OK;
// m_inputClient is a winrt::com_ptr<IAudioClient3>
if (m_inputClient) {
hr = m_inputClient->GetMixFormat(&captureFormat);
// m_audioCaptureClient is a winrt::com_ptr<IAudioCaptureClient>
if (!m_audioCaptureClient) {
hr = m_inputClient->Initialize(
AUDCLNT_SHAREMODE_SHARED,
AUDCLNT_STREAMFLAGS_EVENTCALLBACK,
0,
0,
captureFormat,
nullptr);
hr = m_inputClient->GetService(__uuidof(IAudioCaptureClient), m_audioCaptureClient.put_void());
hr = m_inputClient->SetEventHandle(m_inputReadyEvent.get());
hr = m_inputClient->Reset();
hr = m_inputClient->Start();
}
}
hr = m_inputClient->GetBufferSize(&inputBufferSize);
// multiplying the buffer size by the number of channels
inputBufferSize *= 2;
// m_outputClient is a winrt::com_ptr<IAudioClient3>
if (m_outputClient) {
hr = m_outputClient->GetMixFormat(&renderFormat);
// m_audioRenderClientis a winrt::com_ptr<IAudioRenderClient>
if (!m_audioRenderClient) {
hr = m_outputClient->Initialize(
AUDCLNT_SHAREMODE_SHARED,
AUDCLNT_STREAMFLAGS_EVENTCALLBACK,
0,
0,
captureFormat,
nullptr);
hr = m_outputClient->GetService(__uuidof(IAudioRenderClient), m_audioRenderClient.put_void());
hr = m_outputClient->SetEventHandle(m_outputReadyEvent.get());
hr = m_outputClient->Reset();
hr = m_outputClient->Start();
}
}
hr = m_outputClient->GetBufferSize(&outputBufferSize);
// multiplying the buffer size by the number of channels
outputBufferSize *= 2;
while (m_isRunning)
{
// ===== INPUT =====
// waiting for the capture event
WaitForSingleObject(m_inputReadyEvent.get(), INFINITE);
// getting the input buffer data
hr = m_audioCaptureClient->GetNextPacketSize(&bufferFrameCount);
while (SUCCEEDED(hr) && bufferFrameCount > 0) {
m_audioCaptureClient->GetBuffer(&streamBuffer, &bufferFrameCount, &captureFlags, nullptr, nullptr);
if (bufferFrameCount != 0) {
captureBuffer.write(reinterpret_cast<float*>(streamBuffer), bufferFrameCount * 2);
hr = m_audioCaptureClient->ReleaseBuffer(bufferFrameCount);
if (FAILED(hr)) {
m_audioCaptureClient->ReleaseBuffer(0);
}
}
else
{
m_audioCaptureClient->ReleaseBuffer(0);
}
hr = m_audioCaptureClient->GetNextPacketSize(&bufferFrameCount);
}
// ===== CALLBACK =====
auto size = captureBuffer.size();
float* userInputData = (float*)calloc(size, sizeof(float));
float* userOutputData = (float*)calloc(size, sizeof(float));
captureBuffer.read(userInputData, size);
OnData(userInputData, userOutputData, size / 2, 2, 48000);
renderBuffer.write(userOutputData, size);
free(userInputData);
free(userOutputData);
// ===== OUTPUT =====
// waiting for the render event
WaitForSingleObject(m_outputReadyEvent.get(), INFINITE);
// getting information about the output buffer
hr = m_outputClient->GetBufferSize(&bufferFrameCount);
hr = m_outputClient->GetCurrentPadding(&numFramesPadding);
// adjust the frame count with the padding
bufferFrameCount -= numFramesPadding;
if (bufferFrameCount != 0) {
hr = m_audioRenderClient->GetBuffer(bufferFrameCount, &streamBuffer);
auto count = (bufferFrameCount * 2);
if (renderBuffer.read(reinterpret_cast<float*>(streamBuffer), count) < count) {
// captureBuffer is not full enough, we should fill the remainder with 0
}
hr = m_audioRenderClient->ReleaseBuffer(bufferFrameCount, 0);
if (FAILED(hr)) {
m_audioRenderClient->ReleaseBuffer(0, 0);
}
}
else
{
m_audioRenderClient->ReleaseBuffer(0, 0);
}
}
exit:
// Cleanup code
}
I removed the error handling code for clarity, most of it is :
if (FAILED(hr))
goto exit;
#IInspectable was right, there's something wrong with my code : the audio processing is done by a library which then calls callbacks with some results.
In my callback, I try to raise a winrt::event, but it sometimes takes more than 50ms. When it happens, it blocks the audio thread, and creates discontinuity...
Related
I'm working on a lowest-possible latency MIDI synthetizer software. I'm aware of ASIO and other alternatives, but as they have apparently made significant improvements to the WASAPI stack (in shared mode, at least), I'm curious to try it out. I first wrote a simple event-driven version of program, but as that's not the recommended way to do low-latency audio on Windows 10 (according to the docs), I'm trying to migrate to the Real-Time Work Queue API.
The documentation on Low Latency Audio states that it is recommended to use the Real-Time Work Queue API or MFCreateMFByteStreamOnStreamEx with WASAPI in order for the OS to manage work items in a way that will avoid interference from non-audio subsystems. This seems like a good idea, but the latter option seems to require some managed code (demonstrated in this WindowsAudioSession example), which I know nothing about and would preferably avoid (also the header Robytestream.h which has defs for the IRandomAccessStream isn't found on my system either).
The RTWQ example included in the docs is incomplete (doesn't compile as such), and I have made the necessary additions to make it compilable:
class my_rtqueue : IRtwqAsyncCallback {
public:
IRtwqAsyncResult* pAsyncResult;
RTWQWORKITEM_KEY workItemKey;
DWORD WorkQueueId;
STDMETHODIMP GetParameters(DWORD* pdwFlags, DWORD* pdwQueue)
{
HRESULT hr = S_OK;
*pdwFlags = 0;
*pdwQueue = WorkQueueId;
return hr;
}
//-------------------------------------------------------
STDMETHODIMP Invoke(IRtwqAsyncResult* pAsyncResult)
{
HRESULT hr = S_OK;
IUnknown* pState = NULL;
WCHAR className[20];
DWORD bufferLength = 20;
DWORD taskID = 0;
LONG priority = 0;
BYTE* pData;
hr = render_info.renderclient->GetBuffer(render_info.buffer_framecount, &pData);
ERROR_EXIT(hr);
update_buffer((unsigned short*)pData, render_info.framesize_bytes / (2*sizeof(unsigned short))); // 2 channels, sizeof(unsigned short) == 2
hr = render_info.renderclient->ReleaseBuffer(render_info.buffer_framecount, 0);
ERROR_EXIT(hr);
return S_OK;
}
STDMETHODIMP QueryInterface(const IID &riid, void **ppvObject) {
return 0;
}
ULONG AddRef() {
return 0;
}
ULONG Release() {
return 0;
}
HRESULT queue(HANDLE event) {
HRESULT hr;
hr = RtwqPutWaitingWorkItem(event, 1, this->pAsyncResult, &this->workItemKey);
return hr;
}
my_rtqueue() : workItemKey(0) {
HRESULT hr = S_OK;
IRtwqAsyncCallback* callback = NULL;
DWORD taskId = 0;
WorkQueueId = RTWQ_MULTITHREADED_WORKQUEUE;
//WorkQueueId = RTWQ_STANDARD_WORKQUEUE;
hr = RtwqLockSharedWorkQueue(L"Pro Audio", 0, &taskId, &WorkQueueId);
ERROR_THROW(hr);
hr = RtwqCreateAsyncResult(NULL, reinterpret_cast<IRtwqAsyncCallback*>(this), NULL, &pAsyncResult);
ERROR_THROW(hr);
}
int stop() {
HRESULT hr;
if (pAsyncResult)
pAsyncResult->Release();
if (0xFFFFFFFF != this->WorkQueueId) {
hr = RtwqUnlockWorkQueue(this->WorkQueueId);
if (FAILED(hr)) {
printf("Failed with RtwqUnlockWorkQueue 0x%x\n", hr);
return 0;
}
}
return 1;
}
};
And so, the actual WASAPI code (HRESULT error checking is omitted for clarity):
void thread_main(LPVOID param) {
HRESULT hr;
REFERENCE_TIME hnsRequestedDuration = 0;
IMMDeviceEnumerator* pEnumerator = NULL;
IMMDevice* pDevice = NULL;
IAudioClient3* pAudioClient = NULL;
IAudioRenderClient* pRenderClient = NULL;
WAVEFORMATEX* pwfx = NULL;
HANDLE hEvent = NULL;
HANDLE hTask = NULL;
UINT32 bufferFrameCount;
BYTE* pData;
DWORD flags = 0;
hr = RtwqStartup();
// also, hr is checked for errors every step of the way
hr = CoInitialize(NULL);
hr = CoCreateInstance(
CLSID_MMDeviceEnumerator, NULL,
CLSCTX_ALL, IID_IMMDeviceEnumerator,
(void**)&pEnumerator);
hr = pEnumerator->GetDefaultAudioEndpoint(
eRender, eConsole, &pDevice);
hr = pDevice->Activate(
IID_IAudioClient, CLSCTX_ALL,
NULL, (void**)&pAudioClient);
WAVEFORMATEX wave_format = {};
wave_format.wFormatTag = WAVE_FORMAT_PCM;
wave_format.nChannels = 2;
wave_format.nSamplesPerSec = 48000;
wave_format.nAvgBytesPerSec = 48000 * 2 * 16 / 8;
wave_format.nBlockAlign = 2 * 16 / 8;
wave_format.wBitsPerSample = 16;
UINT32 DP, FP, MINP, MAXP;
hr = pAudioClient->GetSharedModeEnginePeriod(&wave_format, &DP, &FP, &MINP, &MAXP);
printf("DefaultPeriod: %u, Fundamental period: %u, min_period: %u, max_period: %u\n", DP, FP, MINP, MAXP);
hr = pAudioClient->InitializeSharedAudioStream(AUDCLNT_STREAMFLAGS_EVENTCALLBACK, MINP, &wave_format, 0);
my_rtqueue* workqueue = NULL;
try {
workqueue = new my_rtqueue();
}
catch (...) {
hr = E_ABORT;
ERROR_EXIT(hr);
}
hr = pAudioClient->GetBufferSize(&bufferFrameCount);
PWAVEFORMATEX wf = &wave_format;
UINT32 current_period;
pAudioClient->GetCurrentSharedModeEnginePeriod(&wf, ¤t_period);
INT32 FrameSize_bytes = bufferFrameCount * wave_format.nChannels * wave_format.wBitsPerSample / 8;
printf("bufferFrameCount: %u, FrameSize_bytes: %d, current_period: %u\n", bufferFrameCount, FrameSize_bytes, current_period);
hr = pAudioClient->GetService(
IID_IAudioRenderClient,
(void**)&pRenderClient);
render_info.framesize_bytes = FrameSize_bytes;
render_info.buffer_framecount = bufferFrameCount;
render_info.renderclient = pRenderClient;
hEvent = CreateEvent(nullptr, false, false, nullptr);
if (hEvent == INVALID_HANDLE_VALUE) { ERROR_EXIT(0); }
hr = pAudioClient->SetEventHandle(hEvent);
const size_t num_samples = FrameSize_bytes / sizeof(unsigned short);
DWORD taskIndex = 0;
hTask = AvSetMmThreadCharacteristics(TEXT("Pro Audio"), &taskIndex);
if (hTask == NULL) {
hr = E_FAIL;
}
hr = pAudioClient->Start(); // Start playing.
running = 1;
while (running) {
workqueue->queue(hEvent);
}
workqueue->stop();
hr = RtwqShutdown();
delete workqueue;
running = 0;
return 1;
}
This seems to kind of work (ie. audio is being output), but on every other invocation of my_rtqueue::Invoke(), IAudioRenderClient::GetBuffer() returns a HRESULT of 0x88890006 (-> AUDCLNT_E_BUFFER_TOO_LARGE), and the actual audio output is certainly not what I intend it to be.
What issues are there with my code? Is this the right way to use RTWQ with WASAPI?
Turns out there were a number of issues with my code, none of which had really anything to do with Rtwq. The biggest issue was me assuming that the shared mode audio stream was using 16-bit integer samples, when in reality my audio was setup for 32-bit float format (WAVE_FORMAT_IEEE_FLOAT). The currently active shared mode format, period etc. should be fetched like this:
WAVEFORMATEX *wavefmt = NULL;
UINT32 current_period = 0;
hr = pAudioClient->GetCurrentSharedModeEnginePeriod((WAVEFORMATEX**)&wavefmt, ¤t_period);
wavefmt now contains the output format info of the current shared mode. If the wFormatTag field is equal to WAVE_FORMAT_EXTENSIBLE, one needs to cast WAVEFORMATEX to WAVEFORMATEXTENSIBLE to see what the actual format is. After this, one needs to fetch the minimum period supported by the current setup, like so:
UINT32 DP, FP, MINP, MAXP;
hr = pAudioClient->GetSharedModeEnginePeriod(wavefmt, &DP, &FP, &MINP, &MAXP);
and then initialize the audio stream with the new InitializeSharedAudioStream function:
hr = pAudioClient->InitializeSharedAudioStream(AUDCLNT_STREAMFLAGS_EVENTCALLBACK, MINP, wavefmt, NULL);
... get the buffer's actual size:
hr = pAudioClient->GetBufferSize(&render_info.buffer_framecount);
and use GetCurrentPadding in the Get/ReleaseBuffer logic:
UINT32 pad = 0;
hr = render_info.audioclient->GetCurrentPadding(&pad);
int actual_size = (render_info.buffer_framecount - pad);
hr = render_info.renderclient->GetBuffer(actual_size, &pData);
if (SUCCEEDED(hr)) {
update_buffer((float*)pData, actual_size);
hr = render_info.renderclient->ReleaseBuffer(actual_size, 0);
ERROR_EXIT(hr);
}
The documentation for IAudioClient::Initialize states the following about shared mode streams (I assume it also applies to the new IAudioClient3):
Each time the thread awakens, it should call IAudioClient::GetCurrentPadding to determine how much data to write to a rendering buffer or read from a capture buffer. In contrast to the two buffers that the Initialize method allocates for an exclusive-mode stream that uses event-driven buffering, a shared-mode stream requires a single buffer.
Using GetCurrentPadding solves the problem with AUDCLNT_E_BUFFER_TOO_LARGE, and feeding the buffer with 32-bit float samples instead of 16-bit integers makes the output sound fine on my system (although the effect was quite funky!).
If someone comes up with better/more correct ways to use the Rtwq API, I would love to hear them.
I am implementing a Windows Biometric Driver using the umdf sample from github.
When I call WinBioCaptureSample the next plugin's methods run in a loop.
SensorAdapterClearContext
EngineAdapterClearContext
SensorAdapterStartCapture
SensorAdapterFinishCapture
I used TraceView to debug my driver and it shows the next trace messages when is stuck in a loop.
00000001 driver 352840 439560 1 1 04\05\2018-16:46:13:12 CBiometricDevice::OnGetSensorStatus Called.
00000002 driver 352840 439560 1 2 04\05\2018-16:46:13:12 CBiometricDevice::OnGetAttributes Called.
00000003 driver 352840 439560 1 3 04\05\2018-16:46:13:12 CBiometricDevice::OnCaptureDataBuffer too small - must be at least 0x18.
00000004 driver 352840 439560 1 4 04\05\2018-16:46:13:12 CBiometricDevice::OnCaptureData Called.
00000005 driver 352840 439560 4 5 04\05\2018-16:46:13:28 CBiometricDevice::OnGetSensorStatus Called.
00000006 driver 352840 439560 1 6 04\05\2018-16:46:13:29 CBiometricDevice::OnCaptureDataBuffer too small - must be at least 0x18.
00000007 driver 352840 439560 1 7 04\05\2018-16:46:13:29 CBiometricDevice::OnCaptureData Called.
00000008 driver 352840 439560 1 8 04\05\2018-16:46:13:30 CBiometricDevice::OnGetSensorStatus Called.
00000009 driver 352840 439560 4 9 04\05\2018-16:46:13:30 CBiometricDevice::OnCaptureDataBuffer too small - must be at least 0x18.
00000010 driver 352840 439560 1 10 04\05\2018-16:46:13:31 CBiometricDevice::OnCaptureData Called.
...
The method CBiometricDevice::OnGetSensorStatus always returns WINBIO_SENSOR_READY
diagnostics->WinBioHresult = S_OK;
diagnostics->SensorStatus = WINBIO_SENSOR_READY;
MyRequest.SetInformation(diagnostics->PayloadSize);
MyRequest.SetCompletionHr(S_OK);
Next is the method CBiometricDevice::OnCaptureData
DWORD WINAPI
CaptureSleepThread(
LPVOID lpParam
)
{
CBiometricDevice *device = (CBiometricDevice *) lpParam;
PCAPTURE_SLEEP_PARAMS sleepParams = device->GetCaptureSleepParams();
if (sleepParams->SleepValue > 60)
{
sleepParams->SleepValue = 60;
}
Sleep(sleepParams->SleepValue * 1000);
UCHAR szBuffer[] = { 0x08, 0x01, 0x00, 0x02 };
ULONG cbRead = 4;
sleepParams->captureData->WinBioHresult = S_OK;
sleepParams->captureData->SensorStatus = WINBIO_SENSOR_ACCEPT;
sleepParams->captureData->RejectDetail = 0;
sleepParams->captureData->CaptureData.Size = cbRead;
RtlCopyMemory(sleepParams->captureData->CaptureData.Data, szBuffer, cbRead);
device->CompletePendingRequest(sleepParams->Hr, sleepParams->Information);
return 0;
}
CBiometricDevice::OnCaptureData(
_Inout_ IWDFIoRequest *FxRequest
)
{
ULONG controlCode = 0;
PWINBIO_CAPTURE_PARAMETERS captureParams = NULL;
SIZE_T inputBufferSize = 0;
PWINBIO_CAPTURE_DATA captureData = NULL;
SIZE_T outputBufferSize = 0;
bool requestPending = false;
EnterCriticalSection(&m_RequestLock);
if (m_PendingRequest == NULL)
{
if (m_SleepThread != INVALID_HANDLE_VALUE)
{
LeaveCriticalSection(&m_RequestLock);
// TODO: Add code to signal thread to exit.
WaitForSingleObject(m_SleepThread, INFINITE);
CloseHandle(m_SleepThread);
m_SleepThread = INVALID_HANDLE_VALUE;
EnterCriticalSection(&m_RequestLock);
}
if (m_PendingRequest == NULL)
{
m_PendingRequest = FxRequest;
m_PendingRequest->MarkCancelable(this);
}
else
{
requestPending = true;
}
}
else
{
requestPending = true;
}
LeaveCriticalSection(&m_RequestLock);
if (requestPending)
{
FxRequest->Complete(WINBIO_E_DATA_COLLECTION_IN_PROGRESS);
return;
}
GetIoRequestParams(FxRequest,
&controlCode,
(PUCHAR *)&captureParams,
&inputBufferSize,
(PUCHAR *)&captureData,
&outputBufferSize);
if (inputBufferSize < sizeof (WINBIO_CAPTURE_PARAMETERS))
{
TraceEvents(TRACE_LEVEL_ERROR,
BIOMETRIC_TRACE_DEVICE,
"%!FUNC!Invalid argument(s).");
CompletePendingRequest(E_INVALIDARG, 0);
return;
}
if (outputBufferSize < sizeof(DWORD))
{
TraceEvents(TRACE_LEVEL_ERROR,
BIOMETRIC_TRACE_DEVICE,
"%!FUNC!Output buffer NULL or too small to return size information.");
CompletePendingRequest(E_INVALIDARG, 0);
return;
}
if (outputBufferSize < sizeof (WINBIO_CAPTURE_DATA))
{
TraceEvents(TRACE_LEVEL_ERROR,
BIOMETRIC_TRACE_DEVICE,
"%!FUNC!Buffer too small - must be at least 0x%x.", sizeof (WINBIO_CAPTURE_DATA));
DWORD cbSize = 262144;//obtained from MAXIMUM_TRANSFER_SIZE policy of WinUsb
captureData->PayloadSize = (DWORD) sizeof(WINBIO_CAPTURE_DATA) + cbSize;
CompletePendingRequest(S_OK, sizeof(DWORD));
return;
}
RtlZeroMemory(captureData, outputBufferSize);
captureData->PayloadSize = (DWORD) outputBufferSize;// (DWORD) sizeof(WINBIO_CAPTURE_DATA);
captureData->WinBioHresult = WINBIO_E_NO_CAPTURE_DATA;
captureData->SensorStatus = WINBIO_SENSOR_FAILURE;
captureData->RejectDetail= 0;
captureData->CaptureData.Size = 0;
if (captureParams->Purpose == WINBIO_NO_PURPOSE_AVAILABLE)
{
captureData->WinBioHresult = WINBIO_E_UNSUPPORTED_PURPOSE;
}
else if ((captureParams->Format.Type != WINBIO_ANSI_381_FORMAT_TYPE) ||
(captureParams->Format.Owner != WINBIO_ANSI_381_FORMAT_OWNER))
{
captureData->WinBioHresult = WINBIO_E_UNSUPPORTED_DATA_FORMAT;
}
else if (captureParams->Flags != WINBIO_DATA_FLAG_RAW)
{
captureData->WinBioHresult = WINBIO_E_UNSUPPORTED_DATA_TYPE;
}
struct _WINUSB_PIPE_INFORMATION InputPipeInfo, OutputPipeInfo;
m_pIUsbInputPipe->GetInformation(&InputPipeInfo);
m_pIUsbOutputPipe->GetInformation(&OutputPipeInfo);
m_SleepParams.PipeInId = InputPipeInfo.PipeId;
m_SleepParams.PipeOutId = OutputPipeInfo.PipeId;
m_SleepParams.hDeviceHandle = m_pIUsbInterface->GetWinUsbHandle();
m_SleepParams.cbSize = (DWORD) outputBufferSize;
m_SleepParams.SleepValue = 5;
m_SleepParams.Hr = S_OK;
m_SleepParams.Information = captureData->PayloadSize;
m_SleepParams.captureData = captureData;
m_SleepThread = CreateThread(NULL, // default security attributes
0, // use default stack size
CaptureSleepThread, // thread function name
this, // argument to thread function
0, // use default creation flags
NULL); // returns the thread identifier
TraceEvents(TRACE_LEVEL_ERROR,
BIOMETRIC_TRACE_DEVICE,
"%!FUNC! Called.");
}
The methods SensorAdapterStartCapture and SensorAdapterFinishCapture returns S_OK
static HRESULT
WINAPI
SensorAdapterStartCapture(
_Inout_ PWINBIO_PIPELINE Pipeline,
_In_ WINBIO_BIR_PURPOSE Purpose,
_Out_ LPOVERLAPPED *Overlapped
)
{
HRESULT hr = S_OK;
WINBIO_SENSOR_STATUS sensorStatus = WINBIO_SENSOR_FAILURE;
WINBIO_CAPTURE_PARAMETERS captureParameters = { 0 };
BOOL result = TRUE;
DWORD bytesReturned = 0;
// Verify that pointer arguments are not NULL.
if (!ARGUMENT_PRESENT(Pipeline) ||
!ARGUMENT_PRESENT(Purpose) ||
!ARGUMENT_PRESENT(Overlapped))
{
return E_POINTER;
}
// Retrieve the context from the pipeline.
PWINIBIO_SENSOR_CONTEXT sensorContext =
(PWINIBIO_SENSOR_CONTEXT)Pipeline->SensorContext;
// Verify the state of the pipeline.
if (sensorContext == NULL ||
Pipeline->SensorHandle == INVALID_HANDLE_VALUE)
{
return WINBIO_E_INVALID_DEVICE_STATE;
}
*Overlapped = NULL;
// Synchronously retrieve the status.
hr = SensorAdapterQueryStatus(Pipeline, &sensorStatus);
if (FAILED(hr))
{
return hr;
}
// Determine whether the sensor requires calibration.
//if (sensorStatus == WINBIO_SENSOR_NOT_CALIBRATED)
//{
// Call a custom function that sends IOCTLs to
// the sensor to calibrate it. This operation is
// synchronous.
//hr = _SensorAdapterCalibrate(Pipeline);
// Retrieve the status again to determine whether the
// sensor is ready.
//if (SUCCEEDED(hr))
//{
// hr = SensorAdapterQueryStatus(Pipeline, &sensorStatus);
//}
//if (FAILED(hr))
//{
// return hr;
//}
//}
if (sensorStatus == WINBIO_SENSOR_BUSY)
{
return WINBIO_E_DEVICE_BUSY;
}
if (sensorStatus != WINBIO_SENSOR_READY)
{
return WINBIO_E_INVALID_DEVICE_STATE;
}
// Determine whether the data format has been previously determined.
// If it has not, find a format supported by both the engine and
// the sensor.
if ((sensorContext->Format.Owner == 0) &&
(sensorContext->Format.Type == 0))
{
// Retrieve the format preferred by the engine.
hr = Pipeline->EngineInterface->QueryPreferredFormat(
Pipeline,
&sensorContext->Format,
&sensorContext->VendorFormat
);
if (SUCCEEDED(hr))
{
// Call a private function that queries the sensor driver
// and attaches an attribute array to the sensor context.
// This operation is synchronous.
hr = _SensorAdapterGetAttributes(Pipeline);
}
if (SUCCEEDED(hr))
{
// Search the sensor attributes array for the format
// preferred by the engine adapter.
DWORD i = 0;
for (i = 0; i < sensorContext->AttributesBuffer->SupportedFormatEntries; i++)
{
if ((sensorContext->AttributesBuffer->SupportedFormat[i].Owner == sensorContext->Format.Owner) &&
(sensorContext->AttributesBuffer->SupportedFormat[i].Type == sensorContext->Format.Type))
{
break;
}
}
if (i == sensorContext->AttributesBuffer->SupportedFormatEntries)
{
// No match was found. Use the default.
sensorContext->Format.Owner = WINBIO_ANSI_381_FORMAT_OWNER;
sensorContext->Format.Type = WINBIO_ANSI_381_FORMAT_TYPE;
}
}
else
{
return hr;
}
}
// Set up the parameter-input block needed for the IOCTL.
captureParameters.PayloadSize = sizeof(WINBIO_CAPTURE_PARAMETERS);
captureParameters.Purpose = Purpose;
captureParameters.Format.Owner = sensorContext->Format.Owner;
captureParameters.Format.Type = sensorContext->Format.Type;
CopyMemory(&captureParameters.VendorFormat, &sensorContext->VendorFormat, sizeof(WINBIO_UUID));
captureParameters.Flags = WINBIO_DATA_FLAG_RAW;
// Determine whether a buffer has already been allocated for this sensor.
if (sensorContext->CaptureBuffer == NULL)
{
DWORD allocationSize = 0;
sensorContext->CaptureBufferSize = 0;
// This sample assumes that the sensor driver returns
// a fixed-size DWORD buffer containing the required
// size of the capture buffer if it receives a buffer
// that is smaller than sizeof(WINBIO_CAPTURE_DATA).
//
// Call the driver with a small buffer to get the
// allocation size required for this sensor.
//
// Because this operation is asynchronous, you must block
// and wait for it to complete.
result = DeviceIoControl(
Pipeline->SensorHandle,
IOCTL_BIOMETRIC_CAPTURE_DATA,
&captureParameters,
sizeof(WINBIO_CAPTURE_PARAMETERS),
&allocationSize,
sizeof(DWORD),
&bytesReturned,
&sensorContext->Overlapped
);
if (!result && GetLastError() == ERROR_IO_PENDING)
{
SetLastError(ERROR_SUCCESS);
result = GetOverlappedResult(
Pipeline->SensorHandle,
&sensorContext->Overlapped,
&bytesReturned,
TRUE
);
}
if (!result || bytesReturned != sizeof(DWORD))
{
// An error occurred.
hr = _AdapterGetHresultFromWin32(GetLastError());
return hr;
}
// Make sure that you allocate at least the minimum buffer
// size needed to get the payload structure.
if (allocationSize < sizeof(WINBIO_CAPTURE_DATA))
{
allocationSize = sizeof(WINBIO_CAPTURE_DATA);
}
// Allocate the buffer.
sensorContext->CaptureBuffer = (PWINBIO_CAPTURE_DATA)_AdapterAlloc(allocationSize);
if (!sensorContext->CaptureBuffer)
{
sensorContext->CaptureBufferSize = 0;
return E_OUTOFMEMORY;
}
sensorContext->CaptureBufferSize = allocationSize;
}
else
{
// The buffer has already been allocated. Clear the buffer contents.
SensorAdapterClearContext(Pipeline);
}
// Send the capture request. Because this is an asynchronous operation,
// the IOCTL call will return immediately regardless of
// whether the I/O has completed.
result = DeviceIoControl(
Pipeline->SensorHandle,
IOCTL_BIOMETRIC_CAPTURE_DATA,
&captureParameters,
sizeof(WINBIO_CAPTURE_PARAMETERS),
sensorContext->CaptureBuffer,
(DWORD)sensorContext->CaptureBufferSize,
&bytesReturned,
&sensorContext->Overlapped
);
if (result ||
(!result && GetLastError() == ERROR_IO_PENDING))
{
*Overlapped = &sensorContext->Overlapped;
return S_OK;
}
else
{
hr = _AdapterGetHresultFromWin32(GetLastError());
return hr;
}
}
static HRESULT
WINAPI
SensorAdapterFinishCapture(
_Inout_ PWINBIO_PIPELINE Pipeline,
_Out_ PWINBIO_REJECT_DETAIL RejectDetail
)
{
HRESULT hr = S_OK;
//WINBIO_SENSOR_STATUS sensorStatus = WINBIO_SENSOR_FAILURE;
WINBIO_CAPTURE_PARAMETERS captureParameters = { 0 };
BOOL result = TRUE;
DWORD bytesReturned = 0;
// Verify that pointer arguments are not NULL.
if (!ARGUMENT_PRESENT(Pipeline) ||
!ARGUMENT_PRESENT(RejectDetail))
{
return E_POINTER;
}
// Retrieve the context from the pipeline.
PWINIBIO_SENSOR_CONTEXT sensorContext =
(PWINIBIO_SENSOR_CONTEXT)Pipeline->SensorContext;
// Verify the state of the pipeline.
if (sensorContext == NULL ||
Pipeline->SensorHandle == INVALID_HANDLE_VALUE)
{
return WINBIO_E_INVALID_DEVICE_STATE;
}
// Initialize the RejectDetail argument.
*RejectDetail = 0;
// Wait for I/O completion. This sample assumes that the I/O operation was
// started using the code example shown in the SensorAdapterStartCapture
// documentation.
SetLastError(ERROR_SUCCESS);
result = GetOverlappedResult(
Pipeline->SensorHandle,
&sensorContext->Overlapped,
&bytesReturned,
TRUE
);
if (!result)
{
// There was an I/O error.
return _AdapterGetHresultFromWin32(GetLastError());
}
if (bytesReturned == sizeof(DWORD))
{
// The buffer is not large enough. This can happen if a device needs a
// bigger buffer depending on the purpose. Allocate a larger buffer and
// force the caller to reissue their I/O request.
DWORD allocationSize = sensorContext->CaptureBuffer->PayloadSize;
// Allocate at least the minimum buffer size needed to retrieve the
// payload structure.
if (allocationSize < sizeof(WINBIO_CAPTURE_DATA))
{
allocationSize = sizeof(WINBIO_CAPTURE_DATA);
}
// Free the old buffer and allocate a new one.
_AdapterRelease(sensorContext->CaptureBuffer);
sensorContext->CaptureBuffer = NULL;
sensorContext->CaptureBuffer =
(PWINBIO_CAPTURE_DATA)_AdapterAlloc(allocationSize);
if (sensorContext->CaptureBuffer == NULL)
{
sensorContext->CaptureBufferSize = 0;
return E_OUTOFMEMORY;
}
sensorContext->CaptureBufferSize = allocationSize;
return WINBIO_E_BAD_CAPTURE;
}
// Normalize the status value before sending it back to the biometric service.
if (sensorContext->CaptureBuffer != NULL &&
sensorContext->CaptureBufferSize >= sizeof(WINBIO_CAPTURE_DATA))
{
switch (sensorContext->CaptureBuffer->SensorStatus)
{
case WINBIO_SENSOR_ACCEPT:
{
// The capture was acceptable.
DWORD cbRead = sensorContext->CaptureBuffer->CaptureData.Size;
UCHAR * data = sensorContext->CaptureBuffer->CaptureData.Data;
//wprintf(L"%d ", cbRead);
break;
}
case WINBIO_SENSOR_REJECT:
// The capture was not acceptable. Overwrite the WinBioHresult value
// in case it has not been properly set.
sensorContext->CaptureBuffer->WinBioHresult = WINBIO_E_BAD_CAPTURE;
break;
case WINBIO_SENSOR_BUSY:
// The device is busy. Reset the WinBioHresult value in case it
// has not been properly set.
sensorContext->CaptureBuffer->WinBioHresult = WINBIO_E_DEVICE_BUSY;
break;
case WINBIO_SENSOR_READY:
case WINBIO_SENSOR_NOT_CALIBRATED:
case WINBIO_SENSOR_FAILURE:
default:
// There has been a device failure. Reset the WinBioHresult value
// in case it has not been properly set.
sensorContext->CaptureBuffer->WinBioHresult = WINBIO_E_INVALID_DEVICE_STATE;
break;
}
*RejectDetail = sensorContext->CaptureBuffer->RejectDetail;
hr = sensorContext->CaptureBuffer->WinBioHresult;
}
else
{
// The buffer is not large enough or the buffer pointer is NULL.
hr = WINBIO_E_INVALID_DEVICE_STATE;
}
return hr;
}
I used the next code from this github project
HRESULT CaptureSample()
{
HRESULT hr = S_OK;
WINBIO_SESSION_HANDLE sessionHandle = NULL;
WINBIO_UNIT_ID unitId = 0;
WINBIO_REJECT_DETAIL rejectDetail = 0;
PWINBIO_BIR sample = NULL;
SIZE_T sampleSize = 0;
// Connect to the system pool.
hr = WinBioOpenSession(
WINBIO_TYPE_FINGERPRINT, // Service provider
WINBIO_POOL_SYSTEM, // Pool type
WINBIO_FLAG_RAW, // Access: Capture raw data //To call WinBioCaptureSample function successfully, you must open the session handle by specifying WINBIO_FLAG_RAW
//WINBIO_FLAG_RAW: The client application captures raw biometric data using WinBioCaptureSample.
NULL, // Array of biometric unit IDs //NULL if the PoolType parameter is WINBIO_POOL_SYSTEM
0, // Count of biometric unit IDs//zero if the PoolType parameter is WINBIO_POOL_SYSTEM.
WINBIO_DB_DEFAULT, // Default database
&sessionHandle // [out] Session handle
);
if(FAILED(hr))
{
std::cout << "WinBioOpenSession failed. hr = 0x" << std::hex << hr << std::dec << "\n";
if(sample != NULL)
{
WinBioFree(sample);
sample = NULL;
}
if(sessionHandle != NULL)
{
WinBioCloseSession(sessionHandle);
sessionHandle = NULL;
}
return hr;
}
// Capture a biometric sample.
std::cout << "Calling WinBioCaptureSample - Swipe sensor...\n";
hr = WinBioCaptureSample(
sessionHandle,
WINBIO_NO_PURPOSE_AVAILABLE,
WINBIO_DATA_FLAG_RAW,//WINBIO_DATA_FLAG_RAW
&unitId,
&sample,
&sampleSize,
&rejectDetail
);
if(FAILED(hr))
{
if(hr == WINBIO_E_BAD_CAPTURE)
std:: cout << "Bad capture; reason: " << rejectDetail << "\n";
else
std::cout << "WinBioCaptureSample failed.hr = 0x" << std::hex << hr << std::dec << "\n";
if(sample != NULL)
{
WinBioFree(sample);
sample = NULL;
}
if(sessionHandle != NULL)
{
WinBioCloseSession(sessionHandle);
sessionHandle = NULL;
}
return hr;
}
std::cout << "Swipe processed - Unit ID: " << unitId << "\n";
std::cout << "Captured " << sampleSize << " bytes.\n";
if(sample != NULL)
{
WinBioFree(sample);
sample = NULL;
}
if(sessionHandle != NULL)
{
WinBioCloseSession(sessionHandle);
sessionHandle = NULL;
}
return hr;
}
int main()
{
EnumerateSensors();
CreateDirectoryA("data", NULL);
CaptureSample();
}
Sometimes my code is stuck in a loop and other times is not :(
Any hint is welcomed Thanks.
It seems I might have found a temporary solution to my problem.
First what I did is to change the sensor mode from basic to advanced.
[DriverPlugInAddReg]
HKR,WinBio\Configurations,DefaultConfiguration,,"0"
HKR,WinBio\Configurations\0,SensorMode,0x10001,2 ; Basic - 1, Advanced - 2
And the loop won't start
But I also noticed If I comment the call to the Sleep method inside CaptureSleepThread the loop starts again.
My guess is that Windows Biometric Service is expecting the method CaptureSleepThread to take some time to be considered successful but if the method ends very fast it is considered to be failed despite of the successful responses S_OK and WINBIO_SENSOR_ACCEPT and Windows Biometric Service will retry again calling to SensorAdapterStartCapture causing a loop.
DWORD WINAPI
CaptureSleepThread(
LPVOID lpParam
)
{
CBiometricDevice *device = (CBiometricDevice *) lpParam;
PCAPTURE_SLEEP_PARAMS sleepParams = device->GetCaptureSleepParams();
//
// 1 minute or half a minute delay is the trick.
//
if (sleepParams->SleepValue > 60)
{
sleepParams->SleepValue = 60;
}
Sleep(sleepParams->SleepValue * 1000);
UCHAR szBuffer[] = { 0x08, 0x01, 0x00, 0x02 };
ULONG cbRead = 4;
sleepParams->captureData->WinBioHresult = S_OK;
sleepParams->captureData->SensorStatus = WINBIO_SENSOR_ACCEPT;
sleepParams->captureData->RejectDetail = 0;
sleepParams->captureData->CaptureData.Size = cbRead;
RtlCopyMemory(sleepParams->captureData->CaptureData.Data, szBuffer, cbRead);
device->CompletePendingRequest(sleepParams->Hr, sleepParams->Information);
return 0;
}
Other things I observed can cause a loop is when the call fails and not proper response is set.
//No delay is going to cause a loop
//Sleep(sleepParams->SleepValue * 1000);
//zeroes buffer is considered failed
UCHAR szBuffer[] = { 0x00, 0x00, 0x00, 0x00 };
//zero size is a failed read
ULONG cbRead = 0;
//returning other than S_FALSE or S_OK will cause a loop
sleepParams->captureData->WinBioHresult = S_FALSE;
sleepParams->captureData->SensorStatus = WINBIO_SENSOR_ACCEPT;
sleepParams->captureData->RejectDetail = 0;
sleepParams->captureData->CaptureData.Size = cbRead;
//It is going to fail if CaptureData.Data is empty
//RtlCopyMemory(sleepParams->captureData->CaptureData.Data, szBuffer, cbRead);
Also attaching a debugger like WinDbg or Visual Studio will cause a loop so is better to debug using only Trace messages and TraceView tool and attach a debugger when necessary and a loop will be expected in this case, just ignore it.
I'm using Desktop Duplication from Windows API.
Here is the code to access next frame and get the rectangle of pixels that have change from previous frame.
//
// Get next frame and write it into Data
//
_Success_(*Timeout == false && return == DUPL_RETURN_SUCCESS)
DUPL_RETURN DUPLICATIONMANAGER::GetFrame(_Out_ FRAME_DATA* Data, _Out_ bool* Timeout)
{
IDXGIResource* DesktopResource = nullptr;
DXGI_OUTDUPL_FRAME_INFO FrameInfo;
// Get new frame
HRESULT hr = m_DeskDupl->AcquireNextFrame(10000, &FrameInfo, &DesktopResource);
if (hr == DXGI_ERROR_WAIT_TIMEOUT)
{
*Timeout = true;
return DUPL_RETURN_SUCCESS;
}
*Timeout = false;
if (FAILED(hr))
{
return ProcessFailure(m_Device, L"Failed to acquire next frame in DUPLICATIONMANAGER", L"Error", hr, FrameInfoExpectedErrors);
}
// If still holding old frame, destroy it
if (m_AcquiredDesktopImage)
{
m_AcquiredDesktopImage->Release();
m_AcquiredDesktopImage = nullptr;
}
// QI for IDXGIResource
hr = DesktopResource->QueryInterface(__uuidof(ID3D11Texture2D), reinterpret_cast<void **>(&m_AcquiredDesktopImage));
DesktopResource->Release();
DesktopResource = nullptr;
if (FAILED(hr))
{
return ProcessFailure(nullptr, L"Failed to QI for ID3D11Texture2D from acquired IDXGIResource in DUPLICATIONMANAGER", L"Error", hr);
}
// Get metadata
if (FrameInfo.TotalMetadataBufferSize)
{
// Old buffer too small
if (FrameInfo.TotalMetadataBufferSize > m_MetaDataSize)
{
if (m_MetaDataBuffer)
{
delete [] m_MetaDataBuffer;
m_MetaDataBuffer = nullptr;
}
m_MetaDataBuffer = new (std::nothrow) BYTE[FrameInfo.TotalMetadataBufferSize];
if (!m_MetaDataBuffer)
{
m_MetaDataSize = 0;
Data->MoveCount = 0;
Data->DirtyCount = 0;
return ProcessFailure(nullptr, L"Failed to allocate memory for metadata in DUPLICATIONMANAGER", L"Error", E_OUTOFMEMORY);
}
m_MetaDataSize = FrameInfo.TotalMetadataBufferSize;
}
UINT BufSize = FrameInfo.TotalMetadataBufferSize;
// Get move rectangles
hr = m_DeskDupl->GetFrameMoveRects(BufSize, reinterpret_cast<DXGI_OUTDUPL_MOVE_RECT*>(m_MetaDataBuffer), &BufSize);
if (FAILED(hr))
{
Data->MoveCount = 0;
Data->DirtyCount = 0;
return ProcessFailure(nullptr, L"Failed to get frame move rects in DUPLICATIONMANAGER", L"Error", hr, FrameInfoExpectedErrors);
}
Data->MoveCount = BufSize / sizeof(DXGI_OUTDUPL_MOVE_RECT);
BYTE* DirtyRects = m_MetaDataBuffer + BufSize;
BufSize = FrameInfo.TotalMetadataBufferSize - BufSize;
// Get dirty rectangles
hr = m_DeskDupl->GetFrameDirtyRects(BufSize, reinterpret_cast<RECT*>(DirtyRects), &BufSize);
if (FAILED(hr))
{
Data->MoveCount = 0;
Data->DirtyCount = 0;
return ProcessFailure(nullptr, L"Failed to get frame dirty rects in DUPLICATIONMANAGER", L"Error", hr, FrameInfoExpectedErrors);
}
Data->DirtyCount = BufSize / sizeof(RECT);
Data->MetaData = m_MetaDataBuffer;
}
Data->Frame = m_AcquiredDesktopImage;
Data->FrameInfo = FrameInfo;
//Here I would like to access pixel data from Data->Frame. A buffer of RGBA pixel
return DUPL_RETURN_SUCCESS;
}
Here is Frame_Data structure
typedef struct _FRAME_DATA
{
ID3D11Texture2D* Frame;
DXGI_OUTDUPL_FRAME_INFO FrameInfo;
_Field_size_bytes_((MoveCount * sizeof(DXGI_OUTDUPL_MOVE_RECT)) + (DirtyCount * sizeof(RECT))) BYTE* MetaData;
UINT DirtyCount;
UINT MoveCount;
} FRAME_DATA;
Is it possible to access pixel buffer data that have been modified from Data->Frame
Here is my code to access data :
BYTE* DISPLAYMANAGER::GetImageData(ID3D11Texture2D* texture2D, D3D11_TEXTURE2D_DESC Desc)
{
if (texture2D != NULL)
{
D3D11_TEXTURE2D_DESC description;
texture2D->GetDesc(&description);
description.BindFlags = 0;
description.CPUAccessFlags = D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE;
description.Usage = D3D11_USAGE_STAGING;
description.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
ID3D11Texture2D* texTemp = NULL;
HRESULT hr = m_Device->CreateTexture2D(&description, NULL, &texTemp);
if (FAILED(hr))
{
if (texTemp)
{
texTemp->Release();
texTemp = NULL;
}
return NULL;
}
m_DeviceContext->CopyResource(texTemp, texture2D);
D3D11_MAPPED_SUBRESOURCE mapped;
unsigned int subresource = D3D11CalcSubresource(0, 0, 0);
hr = m_DeviceContext->Map(texTemp, subresource, D3D11_MAP_READ_WRITE, 0, &mapped);
if (FAILED(hr))
{
texTemp->Release();
texTemp = NULL;
return NULL;
}
unsigned char *captureData = new unsigned char[Desc.Width * Desc.Height * 4];
RtlZeroMemory(captureData, Desc.Width * Desc.Height * 4);
const int pitch = mapped.RowPitch;
unsigned char *source = static_cast<unsigned char*>(mapped.pData);
unsigned char *dest = captureData;
for (int i = 0; i < Desc.Height; i++) {
memcpy(captureData, source, Desc.Width * 4);
source += pitch;
captureData += Desc.Width * 4;
}
for (int i = 0; i < Desc.Width * Desc.Height * 4; i++) {
//trace(L"Pixel[%d] = %x\n", i, dest[i]);
}
m_DeviceContext->Unmap(texTemp, 0);
return dest;
}
else
return NULL;
}
Thank you for your help!
The textures you obtain via duplication API are not necessarily accessible for CPU, for individual pixel access. To read the texture data, you might need to create a mappable staging texture and copy the obtained texture there. Then doing the mapping you would get a pointer to actual data. Note that this is, in general, not a performance friendly operation.
You will find related information in other answers as well:
How to work with pixels using Direct2D:
For those times when you absolutely have to do CPU pixel manipulation but still want a substantial degree of acceleration, you can manage your own mappable D3D11 textures. For example, you can use staging textures if you want to asynchronously manipulate your texture resources from the CPU.
Transferring textures across adapters in DirectX 11:
... copies it to a staging resource (created on the same device) using ID3D11DeviceContext::CopyResource. I then map that staging resource with Read...
I'm trying to split up an contiguous buffer into 3 byte channels (RGB). Here is my acutal workflow to get the buffer filled with an image:
Set up an Source Reader (MFVideoFormat_RGB32)
Receive video format information
Read first image and convert to contiguous buffer...
In addition to that, here is the code:
HRESULT hr = S_OK;
IMFAttributes *attributes = NULL;
SafeRelease(&_sourcereader);
hr = MFCreateAttributes(&attributes, 1);
if (FAILED(hr)) {
// TODO: set error
return false;
}
hr = attributes->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, true);
if (FAILED(hr)) {
// TODO: set error
return false;
}
// conversion from qstring to const wchar*
const WCHAR* wfilename = filename.toStdWString().c_str();
// create source reader from file with attributes
hr = MFCreateSourceReaderFromURL(wfilename, attributes, &_sourcereader);
if (FAILED(hr)) {
// TODO: set error
return false;
}
// configure sourcereader for progressive RGB32 frames
IMFMediaType *mediatype = NULL;
hr = MFCreateMediaType(&mediatype);
if (SUCCEEDED(hr))
{
hr = mediatype->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
}
if (SUCCEEDED(hr))
{
hr = mediatype->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32);
}
if (SUCCEEDED(hr))
{
hr = _sourcereader->SetCurrentMediaType(
(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
NULL, mediatype);
}
// Ensure the stream is selected.
if (SUCCEEDED(hr))
{
hr = _sourcereader->SetStreamSelection(
(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, TRUE);
}
if (FAILED(hr)) {
// TODO: Error log for failed configuration
std::cout << "(ConfigureSourceReader) Configuration failed" << std::endl;
return false;
}
//------------------------------------------------------------------
//---------------------- Get Video Format Infos --------------------
//------------------------------------------------------------------
GUID subtype = { 0 };
// Get the media type from the stream.
hr = _sourcereader->GetCurrentMediaType(
(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, &mediatype );
// Make sure it is a video format.
hr = mediatype->GetGUID(MF_MT_SUBTYPE, &subtype);
if (subtype != MFVideoFormat_RGB32)
{
hr = E_UNEXPECTED;
// TODO: Error log message
SafeRelease(&mediatype);
return false;
}
//------------------------------------------------------------------
// Get the width and height
UINT32 width = 0, height = 0;
hr = MFGetAttributeSize(mediatype, MF_MT_FRAME_SIZE, &width, &height);
if (FAILED(hr))
{
// TODO: Error log message
SafeRelease(&mediatype);
return false;
}
//assign dimensions to VideoInfo
_videoinfo.imageHeight = height; _videoinfo.imageWidth = width;
//std::cout << "(GetVideoFormat) width: " << width << ", height: " << height << std::endl;
//------------------------------------------------------------------
//get framerate
UINT32 framerate_num = 0, framerate_denom = 0;
hr = MFGetAttributeRatio(mediatype, MF_MT_FRAME_RATE, &framerate_num, &framerate_denom);
if (FAILED(hr))
{
// TODO: Error log message
SafeRelease(&mediatype);
return false;
}
//set frame rate in struct
_videoinfo.fps = framerate_num / framerate_denom; // TODO: check for valid fps 24,25,30 ...
//------------------------------------------------------------------
// Get length
LONGLONG length = 0;
PROPVARIANT var;
PropVariantInit(&var);
hr = _sourcereader->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE,
MF_PD_DURATION,
&var
);
if (SUCCEEDED(hr)) {
assert(var.vt == VT_UI8);
length = var.hVal.QuadPart;
} else {
// TODO : erro log msg
return false;
}
//Get total framenumber and length: save to info struct
_videoinfo.noofFrames = length / 10000000 * this->getFrameRate(); // incl. conversion from nano sec to sec
_videoinfo.duration = length;
//------------------------------------------------------------------
// Get the stride to find out if the bitmap is top-down or bottom-up.
LONG lStride = 0;
lStride = (LONG)MFGetAttributeUINT32(mediatype, MF_MT_DEFAULT_STRIDE, 1);
_videoinfo.stride = lStride;
_videoinfo.bTopDown = (lStride > 0);
//------------------------------------------------------------------
SafeRelease(&mediatype);
// return true and flag if initialization went well
_bInitialized = true;
return true;
After that I call a function to read a single frame (at the moment the first one).
HRESULT hr = S_OK;
IMFSample *pSample = NULL;
IMFMediaBuffer *buffer = NULL;
DWORD streamIndex, flags;
LONGLONG llTimeStamp;
// Read Sample (RGB32)
hr = _sourcereader->ReadSample (
(DWORD) MF_SOURCE_READER_FIRST_VIDEO_STREAM,
0,
&streamIndex,
&flags,
&llTimeStamp,
&pSample);
if (FAILED (hr)) {
// TODO handle fail case
}
//convert sample data to buffer
hr = pSample->ConvertToContiguousBuffer(&buffer);
if (FAILED (hr)) {
// TODO handle fail case
}
I know that by calling the function buffer->Lock(&pixels, NULL, &nPixels) that I can get the BYTE-stream stored in pixels. In my case I create a custom image with the given height and width (from SourceReader; [first function]). From the empty image I can get an empty color matrix which has to be filled with the following funtion: Color (byte red, byte green, byte blue)
I dont know how to split my RGB32 BYTE array into the single channels to fill my image? Maybe it is a silly question but I am relatively new to this area...
For RGB32 the byte format is:
R=Red
G=Green
B=Blue
A=Transparency
RGBARGBARGBA...
A very simple pseudo-code example of extracting the channels is shown below.
for (int row = 0; row < height; row++) {
for (int col = 0; col < stride; col += 4) {
redBuf[rIndex++] = sample[row * stride + col];
greenBuf[gIndex++] = sample[row * stride + col + 1];
blueBuf[bIndex++] = sample[row * stride + col + 2];
transparencyBuf[tIndex++] = sample[row * stride + col + 3];
}
}
My main goal is to capture 2 audio streams and store them as a vector<BYTE> then come up with a congruence algorithm to check for equality. Right now I am only capturing one stream, however the values of the stream are 0 '/0'. Why am I getting null terminated values for all elements in my BYTE vector?
void AudioDeviceOperator::TakeInput(AudioStreamModel* m)
{
HRESULT hr;
IAudioClient *iac = NULL;
IAudioCaptureClient *pCaptureClient = NULL;
WAVEFORMATEX *mixFormat;
UINT32 bufferFrameCount;
HRESULT de;
de = AudioDeviceEnumerator -> GetDefaultAudioEndpoint(eCapture, eConsole, &SelectedAudioDeviceModel->AudioDevice);
hr = SelectedAudioDeviceModel->AudioDevice -> Activate(IID_IAudioClient, CLSCTX_ALL, NULL, (void**)&iac);
REFERENCE_TIME bufferDuration = 0; //default to min
REFERENCE_TIME periodicity = 0;
GUID trashGuid;
HRESULT tg = CoCreateGuid(&trashGuid);
LPCGUID AudioSessionGuid = &trashGuid;
GUID guid2 = *AudioSessionGuid;
HRESULT guidError = UuidCreate(&guid2); //project -> properties -> Linker -> Command Line -> Rpctr4.lib
iac->GetMixFormat(&mixFormat);
m->StreamFormat = *mixFormat;
if (SUCCEEDED(guidError)) {
cout << "/n" << "Initializing audio stream..";
hr = iac->Initialize(AUDCLNT_SHAREMODE_SHARED, AUDCLNT_STREAMFLAGS_CROSSPROCESS, bufferDuration, periodicity, mixFormat, AudioSessionGuid);
cout << hr;
hr = iac->GetBufferSize(&bufferFrameCount);
cout << hr;
iac->GetService(IID_IAudioCaptureClient, (void**)&pCaptureClient);
// Calculate the actual duration of the allocated buffer.
double hnsActualDuration = (double)REFTIMES_PER_SEC * bufferFrameCount / mixFormat-> nSamplesPerSec;
bool recordAudio = TRUE;
BYTE *sData;
UINT32 numFramesAvailable = 0;
DWORD flags;
UINT32 packetLength = 0;
int numOfPackets = 0;
iac->Start();
while (recordAudio == TRUE)
{
hr = pCaptureClient->GetNextPacketSize(&packetLength);
while (packetLength != 0) {
hr = pCaptureClient->GetBuffer(&sData, &numFramesAvailable, &flags, NULL, NULL);
if (sData != NULL) {
m->Stream.push_back((*sData)); //here is where I write to the vector
numOfPackets++;
}
if (numOfPackets == 100) { // just getting 100 packets for testing
recordAudio = FALSE;
break;
}
}
hr = pCaptureClient->ReleaseBuffer(numFramesAvailable);
}
}
else
cout << "AudioSessionGuidError";
CoTaskMemFree(iac);
AudioDeviceEnumerator->Release();
//pCaptureClient->Release(); // releaseBuffer seeming to release capture client interface as weell.
};
When the audio session starts I make sure to make some noise. With the vector values the way they are I have nothing to compare. I'm also assuming that using that byte vector and rendering it with a IAudioRenderClient will result in nothing, however that is my next plan of action. Any ideas??