Related
I'm trying to resample audio from 44Khz to 48Khz and I'm getting s small light noise after resampling. As if someone is gently ticking the mic. This happens both ways. From 48Khz to 44Khz and vice versa.
I've read that this can happen because swrContext still has some data left and that I shoudl flush the context before resampling next frame. And although this helps a little (less noticeable noise), it's still present.
I've tried using FFmpeg resample filter instead, but the output is just loud incoherent noise. I'm pretty sure that libswresample should not output any noise on resampling which means that I just don't know how to use it well and I'm missing some options.
This is the code for resampler.
int ResampleFrame(VideoState * videoState, AVFrame *decoded_audio_frame, enum AVSampleFormat out_sample_fmt, uint8_t * out_buf)
{
int in_sample_rate = videoState->audio->ptrAudioCodecCtx_->sample_rate;
int out_sample_rate = SAMPLE_RATE;
// get an instance of the AudioResamplingState struct, create if NULL
AudioResamplingState* arState = getAudioResampling(videoState->audio->ptrAudioCodecCtx_->channel_layout);
if (!arState->swr_ctx)
{
printf("swr_alloc error.\n");
return -1;
}
// get input audio channels
arState->in_channel_layout = (videoState->audio->ptrAudioCodecCtx_->channels ==
av_get_channel_layout_nb_channels(videoState->audio->ptrAudioCodecCtx_->channel_layout)) ?
videoState->audio->ptrAudioCodecCtx_->channel_layout :
av_get_default_channel_layout(videoState->audio->ptrAudioCodecCtx_->channels);
// check input audio channels correctly retrieved
if (arState->in_channel_layout <= 0)
{
printf("in_channel_layout error.\n");
return -1;
}
arState->out_channel_layout = AV_CH_LAYOUT_STEREO;
// retrieve number of audio samples (per channel)
arState->in_nb_samples = decoded_audio_frame->nb_samples;
if (arState->in_nb_samples <= 0)
{
printf("in_nb_samples error.\n");
return -1;
}
// Set SwrContext parameters for resampling
av_opt_set_int(arState->swr_ctx, "in_channel_layout", arState->in_channel_layout, 0);
av_opt_set_int(arState->swr_ctx, "in_sample_rate", in_sample_rate, 0);
av_opt_set_sample_fmt(arState->swr_ctx, "in_sample_fmt", videoState->audio->ptrAudioCodecCtx_->sample_fmt, 0);
// Set SwrContext parameters for resampling
av_opt_set_int(arState->swr_ctx, "out_channel_layout", arState->out_channel_layout, 0);
av_opt_set_int(arState->swr_ctx, "out_sample_rate", out_sample_rate, 0);
av_opt_set_sample_fmt(arState->swr_ctx, "out_sample_fmt", out_sample_fmt, 0);
// initialize SWR context after user parameters have been set
int ret = swr_init(arState->swr_ctx);
if (ret < 0)
{
printf("Failed to initialize the resampling context.\n");
return -1;
}
// retrieve output samples number taking into account the progressive delay
int64_t delay = swr_get_delay(arState->swr_ctx, videoState->audio->ptrAudioCodecCtx_->sample_rate) + arState->in_nb_samples;
arState->out_nb_samples = av_rescale_rnd(delay, out_sample_rate, in_sample_rate, AV_ROUND_UP );
// check output samples number was correctly rescaled
if (arState->out_nb_samples <= 0)
{
printf("av_rescale_rnd error\n");
return -1;
}
// get number of output audio channels
arState->out_nb_channels = av_get_channel_layout_nb_channels(arState->out_channel_layout);
// allocate data pointers array for arState->resampled_data and fill data
// pointers and linesize accordingly
// check memory allocation for the resampled data was successful
ret = av_samples_alloc_array_and_samples(&arState->resampled_data, &arState->out_linesize, arState->out_nb_channels, arState->out_nb_samples, out_sample_fmt, 0);
if (ret < 0)
{
printf("av_samples_alloc_array_and_samples() error: Could not allocate destination samples.\n");
return -1;
}
if (arState->swr_ctx)
{
// do the actual audio data resampling
// check audio conversion was successful
int ret_num_samples = swr_convert(arState->swr_ctx,arState->resampled_data,arState->out_nb_samples,(const uint8_t**)decoded_audio_frame->data, decoded_audio_frame->nb_samples);
//int ret_num_samples = swr_convert_frame(arState->swr_ctx,arState->resampled_data,arState->out_nb_samples,(const uint8_t**)decoded_audio_frame->data, decoded_audio_frame->nb_samples);
if (ret_num_samples < 0)
{
printf("swr_convert_error.\n");
return -1;
}
// get the required buffer size for the given audio parameters
// check audio buffer size
arState->resampled_data_size = av_samples_get_buffer_size(&arState->out_linesize, arState->out_nb_channels,ret_num_samples,out_sample_fmt,1);
if (arState->resampled_data_size < 0)
{
printf("av_samples_get_buffer_size error.\n");
return -1;
}
} else {
printf("swr_ctx null error.\n");
return -1;
}
// copy the resampled data to the output buffer
memcpy(out_buf, arState->resampled_data[0], arState->resampled_data_size);
// flush the swr context
int delayed = swr_convert(arState->swr_ctx,arState->resampled_data,arState->out_nb_samples,NULL,0);
if (arState->resampled_data)
{
av_freep(&arState->resampled_data[0]);
}
av_freep(&arState->resampled_data);
arState->resampled_data = NULL;
int ret_data_size = arState->resampled_data_size;
return ret_data_size;
}
I also tries using the filter as shown here but my output is just noise.
This is my filter code
int ResampleFrame(AVFrame *frame, uint8_t *out_buf)
{
/* Push the decoded frame into the filtergraph */
qint32 ret;
ret = av_buffersrc_add_frame_flags(buffersrc_ctx1, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
if (ret < 0)
{
printf("ResampleFrame: Error adding frame to buffer\n");
// Delete input frame and return null
av_frame_unref(frame);
return 0;
}
//printf("resampling\n");
AVFrame *resampled_frame = av_frame_alloc();
/* Pull filtered frames from the filtergraph */
ret = av_buffersink_get_frame(buffersink_ctx1, resampled_frame);
/* Set the timestamp on the resampled frame */
resampled_frame->best_effort_timestamp = resampled_frame->pts;
if (ret < 0)
{
av_frame_unref(frame);
av_frame_unref(resampled_frame);
return 0;
}
int buffer_size = av_samples_get_buffer_size(NULL, 2,resampled_frame->nb_samples,AV_SAMPLE_FMT_S16,1);
memcpy(out_buf,resampled_frame->data,buffer_size);
//av_frame_unref(frame);
av_frame_unref(resampled_frame);
return buffer_size;
}
QString filter_description1 = "aresample=48000,aformat=sample_fmts=s16:channel_layouts=stereo,asetnsamples=n=1024:p=0";
int InitAudioFilter(AVStream *inputStream)
{
char args[512];
int ret;
const AVFilter *buffersrc = avfilter_get_by_name("abuffer");
const AVFilter *buffersink = avfilter_get_by_name("abuffersink");
AVFilterInOut *outputs = avfilter_inout_alloc();
AVFilterInOut *inputs = avfilter_inout_alloc();
filter_graph = avfilter_graph_alloc();
const enum AVSampleFormat out_sample_fmts[] = {AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_NONE};
const int64_t out_channel_layouts[] = {AV_CH_LAYOUT_STEREO, -1};
const int out_sample_rates[] = {48000, -1};
snprintf(args, sizeof(args), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%" PRIx64,
inputStream->codec->time_base.num, inputStream->codec->time_base.den,
inputStream->codec->sample_rate,
av_get_sample_fmt_name(inputStream->codec->sample_fmt),
inputStream->codec->channel_layout);
ret = avfilter_graph_create_filter(&buffersrc_ctx1, buffersrc, "in", args, NULL, filter_graph);
if (ret < 0)
{
printf("InitAudioFilter: Unable to create buffersrc\n");
return -1;
}
ret = avfilter_graph_create_filter(&buffersink_ctx1, buffersink, "out", NULL, NULL, filter_graph);
if (ret < 0)
{
printf("InitAudioFilter: Unable to create buffersink\n");
return ret;
}
// set opt SAMPLE FORMATS
ret = av_opt_set_int_list(buffersink_ctx1, "sample_fmts", out_sample_fmts, -1, AV_OPT_SEARCH_CHILDREN);
if (ret < 0)
{
printf("InitAudioFilter: Cannot set output sample format\n");
return ret;
}
// set opt CHANNEL LAYOUTS
ret = av_opt_set_int_list(buffersink_ctx1, "channel_layouts", out_channel_layouts, -1, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
printf("InitAudioFilter: Cannot set output channel layout\n");
return ret;
}
// set opt OUT SAMPLE RATES
ret = av_opt_set_int_list(buffersink_ctx1, "sample_rates", out_sample_rates, -1, AV_OPT_SEARCH_CHILDREN);
if (ret < 0)
{
printf("InitAudioFilter: Cannot set output sample rate\n");
return ret;
}
/* Endpoints for the filter graph. */
outputs -> name = av_strdup("in");
outputs -> filter_ctx = buffersrc_ctx1;
outputs -> pad_idx = 0;
outputs -> next = NULL;
/* Endpoints for the filter graph. */
inputs -> name = av_strdup("out");
inputs -> filter_ctx = buffersink_ctx1;
inputs -> pad_idx = 0;
inputs -> next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filter_description1.toStdString().c_str(), &inputs, &outputs, NULL)) < 0)
{
printf("InitAudioFilter: Could not add the filter to graph\n");
}
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
{
printf("InitAudioFilter: Could not configure the graph\n");
}
/* Print summary of the sink buffer
* Note: args buffer is reused to store channel layout string */
AVFilterLink *outlink = buffersink_ctx1->inputs[0];
av_get_channel_layout_string(args, sizeof(args), -1, outlink->channel_layout);
QString str = args;
printf("Output: srate:%dHz fmt:%s chlayout: %s\n", (int) outlink->sample_rate,
av_get_sample_fmt_name((AVSampleFormat) outlink->format),
str.toStdString().c_str());
filterGraphInitialized_ = true;
}
And since I don't have much experience with filters or audio for that matter, I'm also probably missing something here. But Can't figure out what.
Thanks
When resizing the images of the swapchain (because the window size changed) I am getting some white blinking. I don't really understand why is the source of this issue.
I have this issue only while using VK_PRESENT_MODE_FIFO_KHR present mode with my Intel(R) UHD Graphics 630 integrated GPU, I don't have this issue with the GeForce GTX 1050. I am find that having different behaviors depending on the GPU really curious with Vulkan.
Maybe the ideal solution for what I try to achieve is to have a swapchain that always do the size of the screen and blit only the visible part if it is possible to do it?
Here is my swapchain resize code (far from optimal as I redo some operations that can be avoided).
bool resize_swapchain(VK_Renderer* renderer, Window* window) {
assert(renderer);
VkResult res;
clear_swapchain(renderer);
// Build the swapchain
// Get the list of VkFormats that are supported:
get_enumeration(vkGetPhysicalDeviceSurfaceFormatsKHR,
VkSurfaceFormatKHR,
surface_formats,
"Failed to get physical device surface formats.\n",
"Found %d surface formats.\n",
renderer->physical_device,
renderer->surface);
// If the format list includes just one entry of VK_FORMAT_UNDEFINED,
// the surface has no preferred format. Otherwise, at least one
// supported format will be returned.
if (surface_formats.size() == 1 && surface_formats[0].format == VK_FORMAT_UNDEFINED) {
renderer->surface_format = VK_FORMAT_B8G8R8A8_UNORM;
} else {
renderer->surface_format = surface_formats[0].format;
}
VkSurfaceCapabilitiesKHR surface_capabilities;
res = vkGetPhysicalDeviceSurfaceCapabilitiesKHR(renderer->physical_device, renderer->surface, &surface_capabilities);
if (res != VK_SUCCESS) {
log(globals.logger, Log_Level::error, "Failed to get physical device surface capabilities.\n");
clear_swapchain(renderer);
return false;
}
get_enumeration(vkGetPhysicalDeviceSurfacePresentModesKHR,
VkPresentModeKHR,
present_modes,
"Failed to get physical device surface present modes.\n",
"Found %d present modes.\n",
renderer->physical_device,
renderer->surface);
// width and height are either both 0xFFFFFFFF, or both not 0xFFFFFFFF.
if (surface_capabilities.currentExtent.width == 0xFFFFFFFF) {
// If the surface size is undefined, the size is set to
// the size of the images requested.
renderer->swapchain_extent.width = window->size.x;
renderer->swapchain_extent.height = window->size.y;
if (renderer->swapchain_extent.width < surface_capabilities.minImageExtent.width) {
renderer->swapchain_extent.width = surface_capabilities.minImageExtent.width;
} else if (renderer->swapchain_extent.width > surface_capabilities.maxImageExtent.width) {
renderer->swapchain_extent.width = surface_capabilities.maxImageExtent.width;
}
if (renderer->swapchain_extent.height < surface_capabilities.minImageExtent.height) {
renderer->swapchain_extent.height = surface_capabilities.minImageExtent.height;
} else if (renderer->swapchain_extent.height > surface_capabilities.maxImageExtent.height) {
renderer->swapchain_extent.height = surface_capabilities.maxImageExtent.height;
}
} else {
// If the surface size is defined, the swap chain size must match
renderer->swapchain_extent = surface_capabilities.currentExtent;
}
// The FIFO present mode is guaranteed by the spec to be supported
#if defined(FL_PROFILING_MODE)
VkPresentModeKHR swapchain_present_mode = VK_PRESENT_MODE_IMMEDIATE_KHR;
#else
VkPresentModeKHR swapchain_present_mode = VK_PRESENT_MODE_FIFO_KHR;
#endif
// Determine the number of VkImage's to use in the swap chain.
// We need to acquire only 1 presentable image at at time.
// Asking for minImageCount images ensures that we can acquire
// 1 presentable image as long as we present it before attempting
// to acquire another.
uint32_t desired_number_of_swapchain_images = surface_capabilities.minImageCount;
VkSurfaceTransformFlagBitsKHR surface_transform;
if (surface_capabilities.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR) {
surface_transform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR;
} else {
surface_transform = surface_capabilities.currentTransform;
}
// Find a supported composite alpha mode - one of these is guaranteed to be set
VkCompositeAlphaFlagBitsKHR composite_alpha_flag = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR;
// TODO change the order if we want to be able to blend the window of our application with the Windows Desktop
VkCompositeAlphaFlagBitsKHR composite_alpha_flags[4] = {
VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR,
VK_COMPOSITE_ALPHA_PRE_MULTIPLIED_BIT_KHR,
VK_COMPOSITE_ALPHA_POST_MULTIPLIED_BIT_KHR,
VK_COMPOSITE_ALPHA_INHERIT_BIT_KHR,
};
for (uint32_t i = 0; i < sizeof(composite_alpha_flags); i++) {
if (surface_capabilities.supportedCompositeAlpha & composite_alpha_flags[i]) {
composite_alpha_flag = composite_alpha_flags[i];
break;
}
}
VkSwapchainCreateInfoKHR swapchain_info = {};
swapchain_info.sType = VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR;
swapchain_info.pNext = nullptr;
swapchain_info.surface = renderer->surface;
swapchain_info.minImageCount = desired_number_of_swapchain_images;
swapchain_info.imageFormat = renderer->surface_format;
swapchain_info.imageExtent.width = renderer->swapchain_extent.width;
swapchain_info.imageExtent.height = renderer->swapchain_extent.height;
swapchain_info.preTransform = surface_transform;
swapchain_info.compositeAlpha = composite_alpha_flag;
swapchain_info.imageArrayLayers = 1;
swapchain_info.presentMode = swapchain_present_mode;
swapchain_info.oldSwapchain = nullptr;
swapchain_info.clipped = true;
swapchain_info.imageColorSpace = VK_COLORSPACE_SRGB_NONLINEAR_KHR;
swapchain_info.imageUsage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
swapchain_info.imageSharingMode = VK_SHARING_MODE_EXCLUSIVE;
swapchain_info.queueFamilyIndexCount = 0;
swapchain_info.pQueueFamilyIndices = nullptr;
uint32_t queue_family_indices[2] = {(uint32_t)renderer->graphics_queue_family_index, (uint32_t)renderer->present_queue_family_index};
if (renderer->graphics_queue_family_index != renderer->present_queue_family_index) {
// If the graphics and present queues are from different queue families,
// we either have to explicitly transfer ownership of images between
// the queues, or we have to create the swapchain with imageSharingMode
// as VK_SHARING_MODE_CONCURRENT
swapchain_info.imageSharingMode = VK_SHARING_MODE_CONCURRENT;
swapchain_info.queueFamilyIndexCount = 2;
swapchain_info.pQueueFamilyIndices = queue_family_indices;
// TODO #Speedup We may want optimize this by using VK_SHARING_MODE_EXCLUSIVE and be explicit about transfert ownership
}
res = vkCreateSwapchainKHR(renderer->device, &swapchain_info, nullptr, &renderer->swapchain);
if (res != VK_SUCCESS) {
log(globals.logger, Log_Level::error, "Failed to create the swapchain.\n");
clear_swapchain(renderer);
return false;
}
log(globals.logger, Log_Level::verbose, "Swapchain created with size (%d, %d).\n",
swapchain_info.imageExtent.width,
swapchain_info.imageExtent.height);
get_enumeration(vkGetSwapchainImagesKHR,
VkImage,
swapchain_images,
"Failed to get swapchain images.\n",
"Found %d swapchain images.\n",
renderer->device,
renderer->swapchain);
renderer->swapchain_buffers.resize(swapchain_images.size());
for (uint32_t i = 0; i < swapchain_images.size(); i++) {
renderer->swapchain_buffers[i].image = swapchain_images[i];
}
for (uint32_t i = 0; i < swapchain_images.size(); i++) {
VkImageViewCreateInfo color_image_view = {};
color_image_view.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
color_image_view.pNext = nullptr;
color_image_view.flags = 0;
color_image_view.image = renderer->swapchain_buffers[i].image;
color_image_view.viewType = VK_IMAGE_VIEW_TYPE_2D;
color_image_view.format = renderer->surface_format;
color_image_view.components.r = VK_COMPONENT_SWIZZLE_R;
color_image_view.components.g = VK_COMPONENT_SWIZZLE_G;
color_image_view.components.b = VK_COMPONENT_SWIZZLE_B;
color_image_view.components.a = VK_COMPONENT_SWIZZLE_A;
color_image_view.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
color_image_view.subresourceRange.baseMipLevel = 0;
color_image_view.subresourceRange.levelCount = 1;
color_image_view.subresourceRange.baseArrayLayer = 0;
color_image_view.subresourceRange.layerCount = 1;
res = vkCreateImageView(renderer->device, &color_image_view, nullptr, &renderer->swapchain_buffers[i].view);
if (res != VK_SUCCESS) {
log(globals.logger, Log_Level::error, "Failed to create image view.\n");
clear_swapchain(renderer);
return false;
}
log(globals.logger, Log_Level::verbose, "Image view %d created.\n", i);
}
// Build the depth buffer
VkImageCreateInfo image_info = {};
const VkFormat depth_format = VK_FORMAT_D32_SFLOAT;
VkFormatProperties format_properties;
bool found_memory_type_index;
vkGetPhysicalDeviceFormatProperties(renderer->physical_device, depth_format, &format_properties);
if (format_properties.linearTilingFeatures & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT) {
image_info.tiling = VK_IMAGE_TILING_LINEAR;
} else if (format_properties.optimalTilingFeatures & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT) {
image_info.tiling = VK_IMAGE_TILING_OPTIMAL;
} else {
// #TODO choose an other format?
log(globals.logger, Log_Level::error, "VK_FORMAT_D32_SFLOAT Unsupported.\n");
clear_swapchain(renderer);
return false;
}
image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
image_info.pNext = nullptr;
image_info.imageType = VK_IMAGE_TYPE_2D;
image_info.format = depth_format;
image_info.extent.width = renderer->swapchain_extent.width;
image_info.extent.height = renderer->swapchain_extent.height;
image_info.extent.depth = 1;
image_info.mipLevels = 1;
image_info.arrayLayers = 1;
image_info.samples = renderer->sample_count_flag;
image_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
image_info.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
image_info.queueFamilyIndexCount = 0;
image_info.pQueueFamilyIndices = nullptr;
image_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
image_info.flags = 0;
VkMemoryAllocateInfo memory_allocation_info = {};
memory_allocation_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
memory_allocation_info.pNext = nullptr;
memory_allocation_info.allocationSize = 0;
memory_allocation_info.memoryTypeIndex = 0;
VkImageViewCreateInfo view_info = {};
view_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
view_info.pNext = nullptr;
view_info.image = nullptr;
view_info.format = depth_format;
view_info.components.r = VK_COMPONENT_SWIZZLE_R;
view_info.components.g = VK_COMPONENT_SWIZZLE_G;
view_info.components.b = VK_COMPONENT_SWIZZLE_B;
view_info.components.a = VK_COMPONENT_SWIZZLE_A;
view_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT;
view_info.subresourceRange.baseMipLevel = 0;
view_info.subresourceRange.levelCount = 1;
view_info.subresourceRange.baseArrayLayer = 0;
view_info.subresourceRange.layerCount = 1;
view_info.viewType = VK_IMAGE_VIEW_TYPE_2D;
view_info.flags = 0;
VkMemoryRequirements memory_requirements;
renderer->depth_buffer.format = depth_format;
/* Create image */
res = vkCreateImage(renderer->device, &image_info, nullptr, &renderer->depth_buffer.image);
if (res != VK_SUCCESS) {
log(globals.logger, Log_Level::error, "Failed to create the depth image.\n");
clear_swapchain(renderer);
return false;
}
vkGetImageMemoryRequirements(renderer->device, renderer->depth_buffer.image, &memory_requirements);
memory_allocation_info.allocationSize = memory_requirements.size;
/* Use the memory properties to determine the type of memory required */
found_memory_type_index = memory_type_from_properties(renderer, memory_requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, &memory_allocation_info.memoryTypeIndex);
if (!found_memory_type_index) {
log(globals.logger, Log_Level::error, "Failed to find memory type to allocate the depth image.\n");
clear_swapchain(renderer);
return false;
}
/* Allocate memory */
res = vkAllocateMemory(renderer->device, &memory_allocation_info, nullptr, &renderer->depth_buffer.memory);
if (res != VK_SUCCESS) {
log(globals.logger, Log_Level::error, "Failed to create memory for depth image.\n");
clear_swapchain(renderer);
return false;
}
/* Bind memory */
res = vkBindImageMemory(renderer->device, renderer->depth_buffer.image, renderer->depth_buffer.memory, 0);
if (res != VK_SUCCESS) {
log(globals.logger, Log_Level::error, "Failed to bind the depth image memory.\n");
clear_swapchain(renderer);
return false;
}
/* Create image view */
view_info.image = renderer->depth_buffer.image;
res = vkCreateImageView(renderer->device, &view_info, nullptr, &renderer->depth_buffer.view);
if (res != VK_SUCCESS) {
log(globals.logger, Log_Level::error, "Failed to create the depth image view.\n");
clear_swapchain(renderer);
return false;
}
log(globals.logger, Log_Level::verbose, "Depth buffer created.\n");
for (size_t i = 0; i < renderer->scenes.size(); i++) {
swapchain_resized(renderer->scenes[i], renderer->swapchain_extent.width, renderer->swapchain_extent.height);
}
return true;
}
Edit: Maybe my issue is more related on how I submit rendered images to the swapchain or the image acquisition.
for (size_t i = 0; i < scene->meshes.size(); i++) {
draw_mesh(scene->meshes[i]);
}
// End the Render pass
vkCmdEndRenderPass(scene->renderer->graphical_command_buffer);
// End command buffer
{
res = vkEndCommandBuffer(scene->renderer->graphical_command_buffer);
}
// Execute queue command buffer
{
/* Queue the command buffer for execution */
const VkCommandBuffer command_buffers[] = {scene->renderer->graphical_command_buffer};
VkFenceCreateInfo fence_create_info;
VkFence draw_fence;
fence_create_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
fence_create_info.pNext = nullptr;
fence_create_info.flags = 0;
vkCreateFence(scene->renderer->device, &fence_create_info, nullptr, &draw_fence);
VkPipelineStageFlags pipe_stage_flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
VkSubmitInfo submit_info[1] = {};
submit_info[0].pNext = nullptr;
submit_info[0].sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit_info[0].waitSemaphoreCount = 1;
submit_info[0].pWaitSemaphores = &scene->image_acquired_semaphore;
submit_info[0].pWaitDstStageMask = &pipe_stage_flags;
submit_info[0].commandBufferCount = 1;
submit_info[0].pCommandBuffers = command_buffers;
submit_info[0].signalSemaphoreCount = 0;
submit_info[0].pSignalSemaphores = nullptr;
/* Queue the command buffer for execution */
res = vkQueueSubmit(scene->renderer->graphics_queue, 1, submit_info, draw_fence);
assert(res == VK_SUCCESS);
/* Now present the image in the window */
VkPresentInfoKHR present;
present.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
present.pNext = nullptr;
present.swapchainCount = 1;
present.pSwapchains = &scene->renderer->swapchain;
present.pImageIndices = &scene->current_buffer;
present.pWaitSemaphores = nullptr;
present.waitSemaphoreCount = 0;
present.pResults = nullptr;
/* Make sure command buffer is finished before presenting */
do {
res = vkWaitForFences(scene->renderer->device, 1, &draw_fence, VK_TRUE, scene->renderer->draw_fence_timeout_us);
} while (res == VK_TIMEOUT);
assert(res == VK_SUCCESS);
res = vkQueuePresentKHR(scene->renderer->present_queue, &present);
assert(res == VK_SUCCESS);
vkDestroyFence(scene->renderer->device, draw_fence, nullptr);
}
On vulkan-tutorial.com it is write that we should also recreate the command buffers with the swapchain (https://vulkan-tutorial.com/Drawing_a_triangle/Swap_chain_recreation), is it really mandatory?
I'm the guy from that reddit thread. I'm not sure 100% if we have the exact same issue, but I can explain what I was dealing with and how I worked around it.
So the issue here has several layers. The first one is that window resizing on Windows blocks the message queue because it needs to capture all the input events for itself. So to work around this you need to make your window update asynchronous, for example through threads.
Now your rendering and window resized work asynchronously, which is great, until someone resizes the windows while you are halfway through rendering the new frame. This IMMEDIATELY causes the swapchain to be VK_ERROR_OUT_OF_DATE_KHR, making you unable to present your render result to the screen. This can cause a variety of different artifacts on the surface, depening on GPU vendor, driver version, and even between different GPUs from the same vendor. This is quite literally undefined behavior. But flickering is definitely one of the common results, where it simply doesn't show anything on the surface until a new successful suspend. So far I have not found a single vendor that supports VK_SUBOPTIMAL_KHR to allow you to keep rendering.
A naive solution would be to give the window full control over the framerate instead, but would give very poor and inconsistent frame timings, especially when going over 60hz. You want the rendering to run as fast as it can, with as little latency as possible.
So before I go into the solution, lets summarize the requirements:
The application (including rendering) doesn't freeze when resizing.
The window does not resize between Acquire and Present
The frame timings are not controlled by the window message queue*
* When not resizing
You might have noticed the asterisk on the last requirement. This is because we will have to make a small compromise. The idea is that we only let the window take control over the frame timings when it is resizing. Outside of that we can draw as fast as possible, since nothing else can invalidate the swapchain in between.
To do this I used Fibers. You can think of fibers as a stack without a thread. You can then jump from the fiber to a different fiber and back. Remember that the message queue (specifically GetMessage/PeekMessage calls) doesn't return when resizing? Well, you can jump out of that loop and back in using fibers! Combined with a timer that causes the switch, we can synchronously update the window as well as render frames. Here is a sample from my code:
LRESULT Window::Impl::WndProc(HWND a_HWND, UINT a_Message, WPARAM a_WParam, LPARAM a_LParam)
{
switch (a_Message)
{
case WM_ENTERSIZEMOVE:
SetTimer(a_HWND, 0, 1, NULL);
break;
case WM_EXITSIZEMOVE:
KillTimer(a_HWND, 0);
break;
case WM_TIMER:
m_MainFiber.Switch();
break;
case WM_MOVE:
if (m_MoveCallback)
{
m_MoveCallback(m_This, Vector2i(static_cast<int16_t>(LOWORD(a_LParam)), static_cast<int16_t>(HIWORD(a_LParam))));
}
break;
case WM_SIZE:
switch (a_WParam)
{
case SIZE_MINIMIZED:
if (m_MinimizeCallback)
{
m_MinimizeCallback(m_This);
}
break;
case SIZE_MAXIMIZED:
if (m_MaximizeCallback)
{
m_MaximizeCallback(m_This);
}
break;
}
if (m_ResizeCallback)
{
m_ResizeCallback(m_This, Vector2i(static_cast<int16_t>(LOWORD(a_LParam)), static_cast<int16_t>(HIWORD(a_LParam))));
}
break;
case WM_CLOSE:
if (m_CloseCallback)
{
m_CloseCallback(m_This);
}
break;
}
if (a_Message == WM_CLOSE)
{
return 0;
}
return DefWindowProcW(a_HWND, a_Message, a_WParam, a_LParam);
}
As you can see, it is actually pretty simple. Start a timer when resize starts, stop it when resize ends, and switch back to the original fiber when it triggers.
Here is the fiber callback itself:
void Window::Impl::FiberCallback()
{
MSG msg;
for (;;)
{
if (PeekMessageW(&msg, m_Window, 0, 0, PM_REMOVE) != 0)
{
TranslateMessage(&msg);
DispatchMessageW(&msg);
}
else
{
m_MainFiber.Switch();
}
}
}
And then the actual polling is as simple as this:
void Window::PollEvents()
{
m_Impl->m_MessageFiber.Switch();
}
This should make PollEvents always return immediately when not resizing, and after the timer expires when you are resizing. It also completely avoids threading since it all runs on the same thread, it just switches between stacks.
Leave a comment if something is unclear, and I hope it solves your problem.
I am trying to integrate the Nvidia Encoder SDK with the Nvidia Frame Buffer Capture SDK. I am capturing frames in a cuda buffer and then sending then to the Nvidia Encoder to be encoded in H.264 format.
The encoded video however, is not what I want it to be, it is a completely pink screen.
Are there any reasons as to why it might be happening?
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <dlfcn.h>
#include <string.h>
#include <getopt.h>
#include <unistd.h>
#include<dlfcn.h>
#include"nvEncodeAPI.h"
#include <NvFBC.h>
// If you have the CUDA Toolkit Installed, you can include the CUDA headers
// #include "cuda.h"
//#include "cuda_drvapi_dynlink_cuda.h"
#include"cuda.h"
#include "NvFBCUtils.h"
#define APP_VERSION 4
#define LIB_NVFBC_NAME "libnvidia-fbc.so.1"
#define LIB_CUDA_NAME "libcuda.so.1"
#define N_FRAMES 10
/*
* CUDA entry points
*/
typedef CUresult (* CUINITPROC) (unsigned int Flags);
typedef CUresult (* CUDEVICEGETPROC) (CUdevice *device, int ordinal);
typedef CUresult (* CUCTXCREATEV2PROC) (CUcontext *pctx, unsigned int flags, CUdevice dev);
typedef CUresult (* CUMEMCPYDTOHV2PROC) (void *dstHost, CUdeviceptr srcDevice, size_t ByteCount);
typedef CUresult (* CUGETDEVICEPTR) (CUdeviceptr *dptr, size_t bytesize);
static CUINITPROC cuInit_ptr = NULL;
static CUDEVICEGETPROC cuDeviceGet_ptr = NULL;
static CUCTXCREATEV2PROC cuCtxCreate_v2_ptr = NULL;
static CUMEMCPYDTOHV2PROC cuMemcpyDtoH_v2_ptr = NULL;
static CUGETDEVICEPTR cuGetDevicePtr=NULL;
typedef NVENCSTATUS (*PNVENCCREATEINSTANCE) (NV_ENCODE_API_FUNCTION_LIST* pFuncList);
/**
* Dynamically opens the CUDA library and resolves the symbols that are
* needed for this application.
*
* \param [out] libCUDA
* A pointer to the opened CUDA library.
*
* \return
* NVFBC_TRUE in case of success, NVFBC_FALSE otherwise.
*/
static NVFBC_BOOL cuda_load_library(void *libCUDA)
{
libCUDA = dlopen(LIB_CUDA_NAME, RTLD_NOW);
if (libCUDA == NULL) {
fprintf(stderr, "Unable to open '%s'\n", LIB_CUDA_NAME);
return NVFBC_FALSE;
}
cuInit_ptr = (CUINITPROC) dlsym(libCUDA, "cuInit");
if (cuInit_ptr == NULL) {
fprintf(stderr, "Unable to resolve symbol 'cuInit'\n");
return NVFBC_FALSE;
}
cuDeviceGet_ptr = (CUDEVICEGETPROC) dlsym(libCUDA, "cuDeviceGet");
if (cuDeviceGet_ptr == NULL) {
fprintf(stderr, "Unable to resolve symbol 'cuDeviceGet'\n");
return NVFBC_FALSE;
}
cuCtxCreate_v2_ptr = (CUCTXCREATEV2PROC) dlsym(libCUDA, "cuCtxCreate_v2");
if (cuCtxCreate_v2_ptr == NULL) {
fprintf(stderr, "Unable to resolve symbol 'cuCtxCreate_v2'\n");
return NVFBC_FALSE;
}
cuMemcpyDtoH_v2_ptr = (CUMEMCPYDTOHV2PROC) dlsym(libCUDA, "cuMemcpyDtoH_v2");
if (cuMemcpyDtoH_v2_ptr == NULL) {
fprintf(stderr, "Unable to resolve symbol 'cuMemcpyDtoH_v2'\n");
return NVFBC_FALSE;
}
cuGetDevicePtr = (CUGETDEVICEPTR) dlsym(libCUDA,"cuMemAlloc");
if(cuGetDevicePtr==NULL){
fprintf(stderr, "Unable to resolve symbol 'cuMemAlloc'\n");
return NVFBC_FALSE;
}
return NVFBC_TRUE;
}
/**
* Initializes CUDA and creates a CUDA context.
*
* \param [in] cuCtx
* A pointer to the created CUDA context.
*
* \return
* NVFBC_TRUE in case of success, NVFBC_FALSE otherwise.
*/
static NVFBC_BOOL cuda_init(CUcontext *cuCtx)
{
CUresult cuRes;
CUdevice cuDev;
cuRes = cuInit_ptr(0);
if (cuRes != CUDA_SUCCESS) {
fprintf(stderr, "Unable to initialize CUDA (result: %d)\n", cuRes);
return NVFBC_FALSE;
}
cuRes = cuDeviceGet_ptr(&cuDev, 0);
if (cuRes != CUDA_SUCCESS) {
fprintf(stderr, "Unable to get CUDA device (result: %d)\n", cuRes);
return NVFBC_FALSE;
}
cuRes = cuCtxCreate_v2_ptr(cuCtx, CU_CTX_SCHED_AUTO, cuDev);
if (cuRes != CUDA_SUCCESS) {
fprintf(stderr, "Unable to create CUDA context (result: %d)\n", cuRes);
return NVFBC_FALSE;
}
return NVFBC_TRUE;
}
/**
* Initializes the NvFBC and CUDA libraries and creates an NvFBC instance.
*
* Creates and sets up a capture session to video memory using the CUDA interop.
*
* Captures a bunch of frames every second, converts them to BMP and saves them
* to the disk.
*/
int main(int argc, char *argv[])
{
//handing the opening of the nvidia encoder library
void* handle = dlopen("libnvidia-encode.so",RTLD_NOW);
if(handle==NULL)
{
printf("Unable to load the nvidia encoder library");
return EXIT_FAILURE;
}
PNVENCCREATEINSTANCE NvEncCreateInstance_ptr;
NvEncCreateInstance_ptr = (PNVENCCREATEINSTANCE) dlsym(handle,"NvEncodeAPICreateInstance");
if(NvEncCreateInstance_ptr==NULL)
{
printf("Failure to load the nv encode lib");
}
static struct option longopts[] = {
{ "get-status", no_argument, NULL, 'g' },
{ "track", required_argument, NULL, 't' },
{ "frames", required_argument, NULL, 'f' },
{ "size", required_argument, NULL, 's' },
{ "format", required_argument, NULL, 'o' },
{ NULL, 0, NULL, 0 }
};
int opt, ret;
unsigned int i, nFrames = N_FRAMES;
NVFBC_SIZE frameSize = { 0, 0 };
NVFBC_BOOL printStatusOnly = NVFBC_FALSE;
NVFBC_TRACKING_TYPE trackingType = NVFBC_TRACKING_DEFAULT;
char outputName[NVFBC_OUTPUT_NAME_LEN];
uint32_t outputId = 0;
void *libNVFBC = NULL, *libCUDA = NULL;
PNVFBCCREATEINSTANCE NvFBCCreateInstance_ptr = NULL;
NVFBC_API_FUNCTION_LIST pFn;
CUcontext cuCtx;
NVFBCSTATUS fbcStatus;
NVFBC_BOOL fbcBool;
NVFBC_SESSION_HANDLE fbcHandle;
NVFBC_CREATE_HANDLE_PARAMS createHandleParams;
NVFBC_GET_STATUS_PARAMS statusParams;
NVFBC_CREATE_CAPTURE_SESSION_PARAMS createCaptureParams;
NVFBC_DESTROY_CAPTURE_SESSION_PARAMS destroyCaptureParams;
NVFBC_DESTROY_HANDLE_PARAMS destroyHandleParams;
NVFBC_TOCUDA_SETUP_PARAMS setupParams;
NVFBC_BUFFER_FORMAT bufferFormat = NVFBC_BUFFER_FORMAT_NV12;
char videoFile[64]="video.h264";
FILE* fd;
/*
* Parse the command line.
*/
while ((opt = getopt_long(argc, argv, "hgt:f:s:o:", longopts, NULL)) != -1) {
switch (opt) {
case 'g':
printStatusOnly = NVFBC_TRUE;
break;
case 't':
NvFBCUtilsParseTrackingType(optarg, &trackingType, outputName);
break;
case 'f':
nFrames = (unsigned int) atoi(optarg);
break;
case 's':
ret = sscanf(optarg, "%ux%u", &frameSize.w, &frameSize.h);
if (ret != 2) {
fprintf(stderr, "Invalid size format: '%s'\n", optarg);
return EXIT_FAILURE;
}
break;
case 'o':
if (!strcasecmp(optarg, "rgb")) {
bufferFormat = NVFBC_BUFFER_FORMAT_RGB;
} else if (!strcasecmp(optarg, "argb")) {
bufferFormat = NVFBC_BUFFER_FORMAT_ARGB;
} else if (!strcasecmp(optarg, "nv12")) {
bufferFormat = NVFBC_BUFFER_FORMAT_NV12;
} else if (!strcasecmp(optarg, "yuv444p")) {
bufferFormat = NVFBC_BUFFER_FORMAT_YUV444P;
} else {
fprintf(stderr, "Unknown buffer format: '%s'\n", optarg);
return EXIT_FAILURE;
}
break;
case 'h':
default:
usage(argv[0]);
return EXIT_SUCCESS;
}
}
NvFBCUtilsPrintVersions(APP_VERSION);
/*
* Dynamically load the NvFBC library.
*/
libNVFBC = dlopen(LIB_NVFBC_NAME, RTLD_NOW);
if (libNVFBC == NULL) {
fprintf(stderr, "Unable to open '%s'\n", LIB_NVFBC_NAME);
return EXIT_FAILURE;
}
fbcBool = cuda_load_library(libCUDA);
if (fbcBool != NVFBC_TRUE) {
return EXIT_FAILURE;
}
fbcBool = cuda_init(&cuCtx);
if (fbcBool != NVFBC_TRUE) {
return EXIT_FAILURE;
}
//input output buffer the contains all the pointers to the functions for the api
NV_ENCODE_API_FUNCTION_LIST nvencFunctionList;
nvencFunctionList.version=NV_ENCODE_API_FUNCTION_LIST_VER;
//this function call populates the passed pointer to the nvidia api function list with
//the function pointers to the routines implemented by the encoder api
NVENCSTATUS status = NvEncCreateInstance_ptr(&nvencFunctionList);//NvEncodeAPICreateInstance(&nvencFunctionList);
//exit code if failed to create instance of the api
if(status!=NV_ENC_SUCCESS)
{
printf("The nvidia encoder could not be initialized");
EXIT_FAILURE;
}
//creating the encode session params
NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openSessionExParams;
//initializing them and then adding the values
// memset(&openSessionExParams,0,sizeof(openSessionExParams));
//setting the version
openSessionExParams.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER;
openSessionExParams.deviceType = NV_ENC_DEVICE_TYPE_CUDA;
//Note - Maybe change the next line
//ading the device pointer which in this case will be a cuda context
openSessionExParams.device = (void*)cuCtx;
openSessionExParams.reserved = NULL;
openSessionExParams.apiVersion = NVENCAPI_VERSION;
//setting all values of the reserved 1 array to 0
//Note - check the next two lines
memset(openSessionExParams.reserved1,0,253*sizeof(uint32_t));//sizeof(openSessionExParams.reserved1)/sizeof(openSessionExParams.reserved1[0]));
memset(openSessionExParams.reserved2,NULL,64*sizeof(void*));//sizeof(openSessionExParams.reserved2)/sizeof(openSessionExParams.reserved2[0]));
//handle for the encoder
void* encoder;
//Note- Seg fault
status = nvencFunctionList.nvEncOpenEncodeSessionEx(&openSessionExParams,&encoder);
//NvEncOpenEncodeSessionEx(&openSessionExParams,encoder);
if(status!=NV_ENC_SUCCESS)
{
printf("Failure opening encoding session: %u\n",status);
return EXIT_FAILURE;
}
//getting the count of the available encoder settings
uint32_t guidCount;
status =nvencFunctionList.nvEncGetEncodeGUIDCount(encoder,&guidCount);
if(status!=NV_ENC_SUCCESS)
{
printf("Failure getting GUID count");
return EXIT_FAILURE;
}
//allocating a buffer that holds that count
GUID* GUIDs = malloc(guidCount*sizeof(GUID));
//number of supported guid
uint32_t supportedGUID;
//adding all the guids to the previously allocated buffer
status =nvencFunctionList.nvEncGetEncodeGUIDs(encoder,GUIDs,guidCount,&supportedGUID);
if(status!=NV_ENC_SUCCESS)
{
printf("Failure filling the guid buffer");
return EXIT_FAILURE;
}
//setting the encode guid
GUID encodeGuid = NV_ENC_CODEC_H264_GUID;
//settin up the presets
uint32_t encodePresetCount;
status =nvencFunctionList.nvEncGetEncodePresetCount(encoder,encodeGuid,&encodePresetCount);
if(status!=NV_ENC_SUCCESS)
{
printf("Failure getting the encoder preset count");
return EXIT_FAILURE;
}
//allocating a buffer to hold the suppoting preset guid
GUID *presetGUIDs = malloc(encodePresetCount*sizeof(GUID));
uint32_t supportedPresetGUIDs;
status =nvencFunctionList.nvEncGetEncodePresetGUIDs(encoder,encodeGuid,presetGUIDs,encodePresetCount,&supportedPresetGUIDs);
if(status!=NV_ENC_SUCCESS)
{
printf("Failure filling up the preset buffer");
return EXIT_FAILURE;
}
//getting a preset guid Note - check this line later
GUID presetGuid = NV_ENC_PRESET_LOW_LATENCY_HQ_GUID;
//setting up a preset congif Note - check this
NV_ENC_PRESET_CONFIG encPresetConfig;
encPresetConfig.version = NV_ENC_PRESET_CONFIG_VER;
encPresetConfig.presetCfg.version=NV_ENC_CONFIG_VER;
status = nvencFunctionList.nvEncGetEncodePresetConfig(encoder,encodeGuid,presetGuid,&encPresetConfig);
if(status!=NV_ENC_SUCCESS)
{
printf("Error getting the preset configuration");
return EXIT_FAILURE;
}
//getting the input format which is YUV for our solution
//retrieving the input formats
uint32_t inputFormatCount;
status = nvencFunctionList.nvEncGetInputFormatCount(encoder,encodeGuid,&inputFormatCount);
if(status!=NV_ENC_SUCCESS)
{
printf("Error getting the input buffer format count");
return EXIT_FAILURE;
}
//allocating a buffer for the input formats
NV_ENC_BUFFER_FORMAT* encodeBufferFormats = malloc(inputFormatCount*sizeof(encodeBufferFormats));
//holding the size of the supported formats
uint32_t supportedInputFormats;
//filling up the above buffer
status = nvencFunctionList.nvEncGetInputFormats(encoder,encodeGuid, encodeBufferFormats,inputFormatCount,&supportedInputFormats);
if(status!=NV_ENC_SUCCESS)
{
printf("Error getting the supported input formats");
return EXIT_FAILURE;
}
//selecting a buffer format
//NV_ENC_BUFFER_FORMAT bufferFormat = NV_ENC_BUFFER_FORMAT_IYUV;
//querying the underlying hardware encoder capabilities can be done using the API
//initializing the hardware encoder
NV_ENC_INITIALIZE_PARAMS initializationParams;
/* NV_ENC_PRESET_CONFIG presetConfig;
presetConfig.version = NV_ENC_PRESET_CONFIG_VER;
NV_ENC_CONFIG encoderConfig;
encoderConfig.version = NV_ENC_CONFIG_VER;
encoderConfig.profileGUID=NV_ENC_CODEC_PROFILE_AUTOSELECT_GUID;
presetConfig.presetCfg=encoderConfig;
status = nvencFunctionList.nvEncGetEncodePresetConfig(encoder,encodeGuid,presetGuid,&presetConfig);*/
if(status!=NV_ENC_SUCCESS)
{
printf("\nUnable to get preset config %u",status);
return EXIT_FAILURE;
}
//initializing the encoder params
initializationParams.version = NV_ENC_INITIALIZE_PARAMS_VER;
initializationParams.encodeConfig = &encPresetConfig.presetCfg;
initializationParams.encodeGUID = encodeGuid;
initializationParams.presetGUID=presetGuid;
initializationParams.encodeWidth = 600;
initializationParams.encodeHeight = 600;
initializationParams.frameRateNum = 60;
initializationParams.frameRateDen = 1;
initializationParams.enablePTD=1;
status = nvencFunctionList.nvEncInitializeEncoder(encoder,&initializationParams);
if(status!=NV_ENC_SUCCESS)
{
printf("Error initializing encoder, %u",status);
return EXIT_FAILURE;
}
//allocating the nvidia input output buffers
int num_macroblocks = ((600 + 15) / 16) * ((600 + 15) / 16);
int max_surfaces = (num_macroblocks >= 8160) ? 16 : 32;
NV_ENC_INPUT_PTR* inputBuffers = (NV_ENC_INPUT_PTR*) malloc(max_surfaces * sizeof(NV_ENC_INPUT_PTR));
NV_ENC_OUTPUT_PTR* outputBuffers = (NV_ENC_OUTPUT_PTR*) malloc(max_surfaces * sizeof(NV_ENC_OUTPUT_PTR));
//CUdeviceptr* cudaDevicePtrs = (CUdeviceptr*) malloc(max_surfaces*sizeof(CUdeviceptr));
for(int i=0;i<max_surfaces;i++)
{
//creating the input buffer
NV_ENC_CREATE_INPUT_BUFFER inputBufferParams;
inputBufferParams.version = NV_ENC_CREATE_INPUT_BUFFER_VER;
inputBufferParams.width =(600 + 31) & ~31;
inputBufferParams.height=(600 + 31) & ~31;
inputBufferParams.bufferFmt = NV_ENC_BUFFER_FORMAT_IYUV;
inputBufferParams.reserved=0;
memset(inputBufferParams.reserved1,0,57*sizeof(uint32_t));
memset(inputBufferParams.reserved2,NULL,63*sizeof(void*));
status = nvencFunctionList.nvEncCreateInputBuffer(encoder,&inputBufferParams);
if(status!=NV_ENC_SUCCESS)
{
printf("Input buffer could not be created");
return EXIT_FAILURE;
}
//creating the output buffer
NV_ENC_CREATE_BITSTREAM_BUFFER bitstreamBufferParams;
bitstreamBufferParams.version = NV_ENC_CREATE_BITSTREAM_BUFFER_VER;
bitstreamBufferParams.reserved=0;
memset(bitstreamBufferParams.reserved1,0,58*sizeof(uint32_t));
memset(bitstreamBufferParams.reserved2,NULL,64*sizeof(void*));
//This line is giving me the error, NV_ENC_ERR_UNIMPLEMENTED
status = nvencFunctionList.nvEncCreateBitstreamBuffer(encoder,&bitstreamBufferParams);
if(status!=NV_ENC_SUCCESS)
{
printf("Bitstream could not be created %i",status);
return EXIT_FAILURE;
}
/* CUresult cuResult = cuGetDevicePtr(&cudaDevicePtrs[i],600);
if(cuResult!=CUDA_SUCCESS)
{
printf("Could not allocate cuda device pointer, %u\n",cuResult);
return EXIT_FAILURE;
}*/
inputBuffers[i]=inputBufferParams.inputBuffer;
outputBuffers[i]=bitstreamBufferParams.bitstreamBuffer;
}
/*
* Resolve the 'NvFBCCreateInstance' symbol that will allow us to get
* the API function pointers.
*/
NvFBCCreateInstance_ptr =
(PNVFBCCREATEINSTANCE) dlsym(libNVFBC, "NvFBCCreateInstance");
if (NvFBCCreateInstance_ptr == NULL) {
fprintf(stderr, "Unable to resolve symbol 'NvFBCCreateInstance'\n");
return EXIT_FAILURE;
}
/*
* Create an NvFBC instance.
*
* API function pointers are accessible through pFn.
*/
memset(&pFn, 0, sizeof(pFn));
pFn.dwVersion = NVFBC_VERSION;
fbcStatus = NvFBCCreateInstance_ptr(&pFn);
if (fbcStatus != NVFBC_SUCCESS) {
fprintf(stderr, "Unable to create NvFBC instance (status: %d)\n",
fbcStatus);
return EXIT_FAILURE;
}
/*
* Create a session handle that is used to identify the client.
*/
memset(&createHandleParams, 0, sizeof(createHandleParams));
createHandleParams.dwVersion = NVFBC_CREATE_HANDLE_PARAMS_VER;
fbcStatus = pFn.nvFBCCreateHandle(&fbcHandle, &createHandleParams);
if (fbcStatus != NVFBC_SUCCESS) {
fprintf(stderr, "%s\n", pFn.nvFBCGetLastErrorStr(fbcHandle));
return EXIT_FAILURE;
}
/*
* Get information about the state of the display driver.
*
* This call is optional but helps the application decide what it should
* do.
*/
memset(&statusParams, 0, sizeof(statusParams));
statusParams.dwVersion = NVFBC_GET_STATUS_PARAMS_VER;
fbcStatus = pFn.nvFBCGetStatus(fbcHandle, &statusParams);
if (fbcStatus != NVFBC_SUCCESS) {
fprintf(stderr, "%s\n", pFn.nvFBCGetLastErrorStr(fbcHandle));
return EXIT_FAILURE;
}
if (printStatusOnly) {
NvFBCUtilsPrintStatus(&statusParams);
return EXIT_SUCCESS;
}
if (statusParams.bCanCreateNow == NVFBC_FALSE) {
fprintf(stderr, "It is not possible to create a capture session "
"on this system.\n");
return EXIT_FAILURE;
}
if (trackingType == NVFBC_TRACKING_OUTPUT) {
if (!statusParams.bXRandRAvailable) {
fprintf(stderr, "The XRandR extension is not available.\n");
fprintf(stderr, "It is therefore not possible to track an RandR output.\n");
return EXIT_FAILURE;
}
outputId = NvFBCUtilsGetOutputId(statusParams.outputs,
statusParams.dwOutputNum,
outputName);
if (outputId == 0) {
fprintf(stderr, "RandR output '%s' not found.\n", outputName);
return EXIT_FAILURE;
}
}
/*
* Create a capture session.
*/
printf("Creating an asynchronous capture session of %u frames with 1 "
"second internal between captures.\n", nFrames);
memset(&createCaptureParams, 0, sizeof(createCaptureParams));
createCaptureParams.dwVersion = NVFBC_CREATE_CAPTURE_SESSION_PARAMS_VER;
createCaptureParams.eCaptureType = NVFBC_CAPTURE_SHARED_CUDA;
createCaptureParams.bWithCursor = NVFBC_TRUE;
createCaptureParams.frameSize = frameSize;
createCaptureParams.eTrackingType = trackingType;
if (trackingType == NVFBC_TRACKING_OUTPUT) {
createCaptureParams.dwOutputId = outputId;
}
fbcStatus = pFn.nvFBCCreateCaptureSession(fbcHandle, &createCaptureParams);
if (fbcStatus != NVFBC_SUCCESS) {
fprintf(stderr, "%s\n", pFn.nvFBCGetLastErrorStr(fbcHandle));
return EXIT_FAILURE;
}
/*
* Set up the capture session.
*/
memset(&setupParams, 0, sizeof(setupParams));
setupParams.dwVersion = NVFBC_TOCUDA_SETUP_PARAMS_VER;
setupParams.eBufferFormat = bufferFormat;
fbcStatus = pFn.nvFBCToCudaSetUp(fbcHandle, &setupParams);
if (fbcStatus != NVFBC_SUCCESS) {
fprintf(stderr, "%s\n", pFn.nvFBCGetLastErrorStr(fbcHandle));
return EXIT_FAILURE;
}
/*
* We are now ready to start grabbing frames.
*/
fd = fopen(videoFile, "wb");
if (fd == NULL)
{
fprintf(stderr,"Unable to create '%s'\n", videoFile);
return EXIT_FAILURE;
}
//configuring the per frame encode parameters
NV_ENC_PIC_PARAMS encoderPicParams;
//Codec specific params
/* NV_ENC_CODEC_PIC_PARAMS h264Params;
h264Params.h264PicParams.reserved3=0;
h264Params.h264PicParams.reservedBitFields=0;
h264Params.h264PicParams.ltrUsageMode=0;
memset(h264Params.h264PicParams.reserved,0,242*sizeof(uint32_t));
memset(h264Params.h264PicParams.reserved2,NULL,61*sizeof(void*));*/
encoderPicParams.version = NV_ENC_PIC_PARAMS_VER;
//encoderPicParams.codecPicParams=h264Params;
encoderPicParams.inputWidth=(600+ 31) & ~31;
encoderPicParams.inputHeight=(600+ 31) & ~31;
encoderPicParams.inputPitch = (600+ 31) & ~31;
//encoderPicParams.encodePicFlags=NV_ENC_PIC_FLAG_FORCEIDR;
//encoderPicParams.bufferFmt=NV_ENC_BUFFER_FORMAT_IYUV;
encoderPicParams.pictureStruct=NV_ENC_PIC_STRUCT_FRAME;
for (i = 0; i < 60; i++) {
static CUdeviceptr cuDevicePtr;
static unsigned char *frame = NULL;
static uint32_t lastByteSize = 0;
//char filename[64];
int index=i;
if(i>=max_surfaces)
{
index=i%max_surfaces;
}
int res;
uint64_t t1, t2, t1_total, t2_total, t_delta, wait_time_ms;
CUresult cuRes;
NVFBC_TOCUDA_GRAB_FRAME_PARAMS grabParams;
NVFBC_FRAME_GRAB_INFO frameInfo;
t1 = NvFBCUtilsGetTimeInMillis();
t1_total = t1;
memset(&grabParams, 0, sizeof(grabParams));
memset(&frameInfo, 0, sizeof(frameInfo));
grabParams.dwVersion = NVFBC_TOCUDA_GRAB_FRAME_PARAMS_VER;
/*
* Use asynchronous calls.
*
* The application will not wait for a new frame to be ready. It will
* capture a frame that is already available. This might result in
* capturing several times the same frame. This can be detected by
* checking the frameInfo.bIsNewFrame structure member.
*/
grabParams.dwFlags = NVFBC_TOCUDA_GRAB_FLAGS_NOWAIT;
/*
* This structure will contain information about the captured frame.
*/
grabParams.pFrameGrabInfo = &frameInfo;
/*
* The frame will be mapped in video memory through this CUDA
* device pointer.
*/
grabParams.pCUDADeviceBuffer = &cuDevicePtr;
/*
* Capture a frame.
*/
fbcStatus = pFn.nvFBCToCudaGrabFrame(fbcHandle, &grabParams);
if (fbcStatus != NVFBC_SUCCESS) {
fprintf(stderr, "%s\n", pFn.nvFBCGetLastErrorStr(fbcHandle));
return EXIT_FAILURE;
}
t2 = NvFBCUtilsGetTimeInMillis();
/*
* Allocate or re-allocate the destination buffer in system memory
* when necessary.
*
* This is to handle change of resolution.
*/
if (lastByteSize < frameInfo.dwByteSize) {
frame = (unsigned char *)realloc(frame, frameInfo.dwByteSize);
if (frame == NULL) {
fprintf(stderr, "Unable to allocate system memory\n");
return EXIT_FAILURE;
}
printf("Reallocated %u KB of system memory\n",
frameInfo.dwByteSize / 1024);
lastByteSize = frameInfo.dwByteSize;
}
printf("%s id %u grabbed in %llu ms",
(frameInfo.bIsNewFrame ? "New frame" : "Frame"),
frameInfo.dwCurrentFrame,
(unsigned long long) (t2 - t1));
/*
* Download frame from video memory to system memory.
*/
t1 = NvFBCUtilsGetTimeInMillis();
cuRes = cuMemcpyDtoH_v2_ptr((void *) frame, cuDevicePtr,
frameInfo.dwByteSize);
if (cuRes != CUDA_SUCCESS) {
fprintf(stderr, "CUDA memcpy failure (result: %d)\n", cuRes);
return EXIT_FAILURE;
}
//creating the params for the nv enc lock input buffer
//inputBufferParams.pSysMemBuffer=(void*) frame;
//status = nvencFunctionList.nvEncCreateInputBuffer(encoder,&inputBufferParams);
//setting up the input params
NV_ENC_REGISTER_RESOURCE registerResource;
registerResource.version=NV_ENC_REGISTER_RESOURCE_VER;
registerResource.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR;
registerResource.width = (600+31)&~31;
registerResource.height=(600+31)&~31;
registerResource.pitch=(600+31)&~31;
registerResource.resourceToRegister = (void*)cuDevicePtr;
registerResource.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12;
status = nvencFunctionList.nvEncRegisterResource(encoder,®isterResource);
if(status!=NV_ENC_SUCCESS)
{
printf("Failed to register resource, %u",status);
return EXIT_FAILURE;
}
//mapping these resources
NV_ENC_MAP_INPUT_RESOURCE mappedResource;
mappedResource.version = NV_ENC_MAP_INPUT_RESOURCE_VER;
mappedResource.registeredResource=registerResource.registeredResource;
status = nvencFunctionList.nvEncMapInputResource(encoder,&mappedResource);
if(status!=NV_ENC_SUCCESS)
{
printf("Could not map resource: %u,",status);
return EXIT_FAILURE;
}
encoderPicParams.outputBitstream=outputBuffers[index];
encoderPicParams.inputBuffer=mappedResource.mappedResource;
encoderPicParams.bufferFmt=mappedResource.mappedBufferFmt;
/* memset(encoderPicParams.reserved1,0,6*sizeof(uint32_t));
memset(encoderPicParams.reserved2,NULL,2*sizeof(void*));
memset(encoderPicParams.reserved3,0,286*sizeof(uint32_t));
memset(encoderPicParams.reserved4,0,60*sizeof(void*));*/
//sending the picture to be encoded
status=nvencFunctionList.nvEncEncodePicture(encoder,&encoderPicParams);
if(status!=NV_ENC_SUCCESS)
{
printf("\n Unable to encode video, Error code: %u",status);
return EXIT_FAILURE;
}
//setting up the lock bitstream params
//sending the data to the output stream
NV_ENC_LOCK_BITSTREAM lockBitstreamParams;
lockBitstreamParams.version=NV_ENC_LOCK_BITSTREAM_VER;
lockBitstreamParams.doNotWait=0;
lockBitstreamParams.outputBitstream=outputBuffers[index];
lockBitstreamParams.reservedBitFields=0;
memset(lockBitstreamParams.reserved,0,219*sizeof(uint32_t));
memset(lockBitstreamParams.reserved1,0,13*sizeof(uint32_t));
memset(lockBitstreamParams.reserved2,NULL,64*sizeof(uint32_t));
status = nvencFunctionList.nvEncLockBitstream(encoder,&lockBitstreamParams);
if(status!=NV_ENC_SUCCESS)
{
printf("error locking bitstream");
return EXIT_FAILURE;
}
//writing the video data to the file
int bytes = fwrite(lockBitstreamParams.bitstreamBufferPtr,1,lockBitstreamParams.bitstreamSizeInBytes ,fd);
printf("\nNumber of bytes: %u\n",bytes);
if(bytes==0)
{
fprintf(stderr, "Unable to write to '%s'\n", videoFile);
return EXIT_FAILURE;
}
status = nvencFunctionList.nvEncUnlockBitstream(encoder,outputBuffers[index]);
status = nvencFunctionList.nvEncUnmapInputResource(encoder,mappedResource.mappedResource);
if(status!=NV_ENC_SUCCESS)
{
printf("failure to map resource, %u",status);
return EXIT_FAILURE;
}
We are trying to implement global shortcuts into our application, developed with Qt 5.5 under Mac OS X.
Unfortunately when adding another library the following error pops up:
/Users/user/app/qxtglobalshortcut5/gui/qxtwindowsystem_mac.cpp:17: error: cannot initialize a variable of type 'CGError' with an rvalue of type '(anonymous enum at /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/usr/include/MacTypes.h:382:1)'
CGError err(noErr);
^ ~~~~~
This is the file in question:
#ifndef QXTWINDOWSYSTEM_MAC_CPP
#define QXTWINDOWSYSTEM_MAC_CPP
#include "qxtwindowsystem.h"
#include "qxtwindowsystem_mac.h"
// WId to return when error
#define WINDOW_NOT_FOUND (WId)(0)
WindowList qxt_getWindowsForPSN(ProcessSerialNumber *psn)
{
static CGSConnection connection = _CGSDefaultConnection();
WindowList wlist;
if (!psn) return wlist;
CGError err(noErr);
// get onnection for given process psn
CGSConnection procConnection;
err = CGSGetConnectionIDForPSN(connection, psn, &procConnection);
if (err != noErr) return wlist;
/* get number of windows open by given process
in Mac OS X an application may have multiple windows, which generally
represent documents. It is also possible that there is no window even
though there is an application, it can simply not have any documents open. */
int windowCount(0);
err = CGSGetOnScreenWindowCount(connection, procConnection, &windowCount);
// if there are no windows open by this application, skip
if (err != noErr || windowCount == 0) return wlist;
// get list of windows
int windowList[windowCount];
int outCount(0);
err = CGSGetOnScreenWindowList(connection, procConnection, windowCount, windowList, &outCount);
if (err != noErr || outCount == 0) return wlist;
for (int i=0; i<outCount; ++i)
{
wlist += windowList[i];
}
return wlist;
}
WindowList QxtWindowSystem::windows()
{
WindowList wlist;
ProcessSerialNumber psn = {0, kNoProcess};
// iterate over list of processes
OSErr err;
while ((err = ::GetNextProcess(&psn)) == noErr)
{
wlist += qxt_getWindowsForPSN(&psn);
}
return wlist;
}
WId QxtWindowSystem::activeWindow()
{
ProcessSerialNumber psn;
OSErr err(noErr);
err = ::GetFrontProcess(&psn);
if (err != noErr) return WINDOW_NOT_FOUND;
// in Mac OS X, first window for given PSN is always the active one
WindowList wlist = qxt_getWindowsForPSN(&psn);
if (wlist.count() > 0)
return wlist.at(0);
return WINDOW_NOT_FOUND;
}
QString QxtWindowSystem::windowTitle(WId window)
{
CGSValue windowTitle;
CGError err(noErr);
static CGSConnection connection = _CGSDefaultConnection();
// This code is so dirty I had to wash my hands after writing it.
// most of CoreGraphics private definitions ask for CGSValue as key but since
// converting strings to/from CGSValue was dropped in 10.5, I use CFString, which
// apparently also works.
// FIXME: Not public API function. Can't compile with OS X 10.8
// err = CGSGetWindowProperty(connection, window, (CGSValue)CFSTR("kCGSWindowTitle"), &windowTitle);
if (err != noErr) return QString();
// this is UTF8 encoded
return QCFString::toQString((CFStringRef)windowTitle);
}
QRect QxtWindowSystem::windowGeometry(WId window)
{
CGRect rect;
static CGSConnection connection = _CGSDefaultConnection();
CGError err = CGSGetWindowBounds(connection, window, &rect);
if (err != noErr) return QRect();
return QRect(rect.origin.x, rect.origin.y, rect.size.width, rect.size.height);
}
/* This method is the only one that is not a complete hack
from Quartz Event Services
http://developer.apple.com/library/mac/#documentation/Carbon/Reference/QuartzEventServicesRef/Reference/reference.html
*/
uint QxtWindowSystem::idleTime()
{
// CGEventSourceSecondsSinceLastEventType returns time in seconds as a double
// also has extremely long name
double idle = 1000 * ::CGEventSourceSecondsSinceLastEventType(kCGEventSourceStateCombinedSessionState, kCGAnyInputEventType);
return (uint)idle;
}
// these are copied from X11 implementation
WId QxtWindowSystem::findWindow(const QString& title)
{
WId result = 0;
WindowList list = windows();
foreach(const WId &wid, list)
{
if (windowTitle(wid) == title)
{
result = wid;
break;
}
}
return result;
}
WId QxtWindowSystem::windowAt(const QPoint& pos)
{
WId result = 0;
WindowList list = windows();
for (int i = list.size() - 1; i >= 0; --i)
{
WId wid = list.at(i);
if (windowGeometry(wid).contains(pos))
{
result = wid;
break;
}
}
return result;
}
#endif // QXTWINDOWSYSTEM_MAC_CPP
The last time we used this library there was no such error and we don't quite understand what the problem is.
The library we added is: https://github.com/ddqd/qxtglobalshortcut5
Thank you all in advance!
I'm trying to create a rendering context for OpenGL 4.3.
I've created a temporary rendering context in order to load all the extensions and then destroyed this temporary rendering context and it works, all the features are loaded.
Now I have to create the final rendering context but it seems I can't find a good pixel format.
This is the function I use to try to find the pixel format and create the final rendering context:
int attributeListInt[19];
int pixelFormat[20];
unsigned int formatCount;
int result;
PIXELFORMATDESCRIPTOR pixelFormatDescriptor;
int attributeList[5];
float fieldOfView, screenAspect;
char *vendorString, *rendererString;
m_hDeviceContext = GetDC(m_hWnd);
if (!m_hDeviceContext)
return false;
attributeListInt[0] = WGL_SUPPORT_OPENGL_ARB;
attributeListInt[1] = TRUE;
attributeListInt[2] = WGL_DRAW_TO_WINDOW_ARB;
attributeListInt[3] = TRUE;
attributeListInt[4] = WGL_ACCELERATION_ARB;
attributeListInt[5] = WGL_FULL_ACCELERATION_ARB;
attributeListInt[6] = WGL_COLOR_BITS_ARB;
attributeListInt[7] = 32;
attributeListInt[8] = WGL_DEPTH_BITS_ARB;
attributeListInt[9] = 24;
attributeListInt[10] = WGL_DOUBLE_BUFFER_ARB;
attributeListInt[11] = TRUE;
attributeListInt[12] = WGL_SWAP_METHOD_ARB;
attributeListInt[13] = WGL_SWAP_EXCHANGE_ARB;
attributeListInt[14] = WGL_PIXEL_TYPE_ARB;
attributeListInt[15] = WGL_TYPE_RGBA_ARB;
attributeListInt[16] = WGL_STENCIL_BITS_ARB;
attributeListInt[17] = 8;
attributeListInt[18] = 0;
result = wglChoosePixelFormatARB(m_hDeviceContext, attributeListInt, NULL, 20, pixelFormat, &formatCount);
if (result != 1)
return false;
bool bCreatedSuccessfully = false;
for (int i = 0; i < formatCount && !bCreatedSuccessfully; i++)
{
result = SetPixelFormat(m_hDeviceContext, pixelFormat[i], &pixelFormatDescriptor);
if (result == 1)
bCreatedSuccessfully = true;
}
if (!bCreatedSuccessfully)
return false;
if (m_bOpenGL43)
{
attributeList[0] = WGL_CONTEXT_MAJOR_VERSION_ARB;
attributeList[1] = 4;
attributeList[2] = WGL_CONTEXT_MINOR_VERSION_ARB;
attributeList[3] = 3;
}
else
{
attributeList[0] = WGL_CONTEXT_MAJOR_VERSION_ARB;
attributeList[1] = 3;
attributeList[2] = WGL_CONTEXT_MINOR_VERSION_ARB;
attributeList[3] = 3;
}
attributeList[4] = 0;
m_hRenderingContext = wglCreateContextAttribsARB(m_hDeviceContext, 0, attributeList);
if (m_hRenderingContext == NULL)
return false;
result = wglMakeCurrent(m_hDeviceContext, m_hRenderingContext);
if (result != 1)
return false;
glClearDepth(1.0f);
glEnable(GL_DEPTH_TEST);
glFrontFace(GL_CW);
glEnable(GL_CULL_FACE);
glCullFace(GL_BACK);
if (bVSyncEnalbed)
result = wglSwapIntervalEXT(1);
else
result = wglSwapIntervalEXT(0);
if (result != 1)
return false;
m_bInitialized = true;
return true;
My problem is that wglChoosePixelFormatARB returns 10 avaiable pixel formats, but none of them seems to be right because SetPixelFormat always return FALSE.
Any idea?