Why image decoded by ffmpeg has a border? - c++

After decoding some videos from iphone, I find the image has a strange border on the right.
It is not black but some random pix (width:4).
why i get the border?
how to cut this border or to turn it to black?
Most of videos dont have this problem.
here is the sample code:
the code will load a video from memory, and decode it
int VideoParseFFmpeg::read_packet(void *opaque, uint8_t *buf, int buf_size) {
struct buffer_data *bd = (struct buffer_data *) opaque;
buf_size = FFMIN(buf_size, bd->size_);
if (!buf_size)
return AVERROR_EOF;
memcpy(buf, bd->ptr_, buf_size);
bd->ptr_ += buf_size;
bd->size_ -= buf_size;
return buf_size;
}
int VideoParseFFmpeg::LoadContent(const std::string &video_content) {
int ret = 0;
...
video_size_ = video_content.size();
avio_ctx_buffer_size_ = video_size_ + AV_INPUT_BUFFER_PADDING_SIZE;
avio_ctx_buffer_ = static_cast<uint8_t *>(av_malloc(avio_ctx_buffer_size_));
bd_.ptr_ = (uint8_t *)(video_content.c_str());
bd_.size_ = video_content.size();
av_register_all();
input_ctx_ = avformat_alloc_context();
avio_ctx_ = avio_alloc_context(avio_ctx_buffer_,
avio_ctx_buffer_size_,
0,
&bd_,
&read_packet,
NULL,
NULL);
if ((ret = av_probe_input_buffer(avio_ctx_, &in_fmt_, "", NULL, 0, 0)) < 0) {
LOGGER_WARN(Log::GetLog(), "fail to prob input, err [{}]", AVERROR(ret));
return -1;
}
input_ctx_->pb = avio_ctx_;
/* open the input file */
if ((ret = avformat_open_input(&input_ctx_, "", in_fmt_, NULL)) != 0) {
LOGGER_WARN(Log::GetLog(), "fail to open input, err [{}]", AVERROR(ret));
return -1;
}
if ((ret = avformat_find_stream_info(input_ctx_, NULL)) < 0) {
LOGGER_WARN(Log::GetLog(),
"fail to find input stream information, err[{}]",
AVERROR(ret));
return -1;
}
/* find the video stream information */
if ((ret = av_find_best_stream(input_ctx_, AVMEDIA_TYPE_VIDEO, -1, -1, &decoder_, 0)) < 0) {
LOGGER_WARN(Log::GetLog(), "fail to find a video stream from input, err[{}]", ret);
return -1;
}
video_stream_idx_ = ret;
if (!(decoder_ctx_ = avcodec_alloc_context3(decoder_))) {
LOGGER_WARN(Log::GetLog(), "fail to alloc avcodec context");
return -1;
}
video_stream_ = input_ctx_->streams[video_stream_idx_];
if ((ret = avcodec_parameters_to_context(decoder_ctx_, video_stream_->codecpar)) < 0) {
LOGGER_WARN(Log::GetLog(), "fail to convert parameters to context, err [{}]", ret);
return -1;
}
if ((ret = avcodec_open2(decoder_ctx_, decoder_, NULL)) < 0) {
LOGGER_WARN(Log::GetLog(), "fail to open decodec, err[{}]", ret);
return -1;
}
return 0;
}
void VideoParseFFmpeg::GetFrames(std::vector<cv::Mat> &frames) {
while (1) {
...
decode_write(decoder_ctx_, &packet_, &buffer, frames);
...
}
//flush
...
}
cv::Mat VideoParseFFmpeg::avFrame2Mat(AVFrame *pAvFrame,
AVCodecContext *pCodecCtx,
AVPixelFormat src_fmt) {
AVFrame *pFrameBGR = av_frame_alloc(); // 存储解码后转换的BGR数据
int w = pAvFrame->width;
int h = pAvFrame->height;
auto size = av_image_get_buffer_size(AV_PIX_FMT_BGR24, w, h, 1);
cv::Mat out(h, w, CV_8UC3);
int ret = av_image_fill_arrays(pFrameBGR->data,
pFrameBGR->linesize,
out.data,
AV_PIX_FMT_BGR24,
w,
h,
1);
img_convert_ctx_ = sws_getCachedContext(img_convert_ctx_,
w,
h,
ConvertDeprecatedFormat(src_fmt),
w,
h,
AV_PIX_FMT_BGR24,
SWS_BICUBIC,
NULL, NULL, NULL);
sws_scale(img_convert_ctx_,
(const uint8_t *const *) pAvFrame->data,
pAvFrame->linesize,
0,
pCodecCtx->height,
pFrameBGR->data,
pFrameBGR->linesize);
av_free(pFrameBGR);
return out;
}
int VideoParseFFmpeg::decode_write(AVCodecContext *avctx,
AVPacket *packet,
uint8_t **buffer,
std::vector<cv::Mat> &p_mat_out) {
AVFrame *frame = NULL, *sw_frame = NULL;
AVFrame *tmp_frame = NULL;
int ret = 0;
ret = avcodec_send_packet(avctx, packet);
...
while (1) {
auto clear = [&frame, &sw_frame, this] {
if (frame != NULL)
av_frame_free(&frame);
if (sw_frame != NULL)
av_frame_free(&sw_frame);
av_packet_unref(&packet_);
};
...
ret = avcodec_receive_frame(avctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
clear();
return 0;
} else if (ret < 0) {
LOGGER_WARN(Log::GetLog(), "error while decoding, err[{}]", AVERROR(ret));
clear();
return ret;
}
tmp_frame = frame;
p_mat_out.push_back(avFrame2Mat(tmp_frame,
avctx,
(AVPixelFormat) tmp_frame->format));
clear();
}
}
After replacing lib tcmalloc with a newer version, the border turns to be a black line (used to be a random line), like this.
there is image content in the area of the border in the original image, how to fix it?

Related

Adding unregistered SEI data to every frame (ffmpeg / C++ / Windows)

I am working with FFMPEG 5.2 using it with C++ in Visual Studio. What I require to do is to add a SEI Unregistered message (5) to every frame of a stream, for that, I am demuxing a MP4 container, then taking the video stream, decoding every packet to get a frame, then add SEI message to every frame, encoding and remuxing a new video stream (video only) and saving the new stream to a separate container.
To add the SEI data I use this specific code:
const char* sideDataMsg = "139FB1A9446A4DEC8CBF65B1E12D2CFDHola";;
size_t sideDataSize = sizeof(sideDataMsg);
AVBufferRef* sideDataBuffer = av_buffer_alloc(20);
sideDataBuffer->data = (uint8_t*)sideDataMsg;
AVFrameSideData* sideData = av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_SEI_UNREGISTERED, sideDataBuffer);
regarding the format of the sideDataMsg I have tried several apporaches including setting it like: "139FB1A9-446A-4DEC-8CBF65B1E12D2CFD+Hola!" which is indicated to be the required format in H.264 specs, however, even when in memory I see the SEI data is added to every frame as we observe as follows:
the resulting stream/container does not shows the expected data, below my entire code, this is mostly code taken/adapted from doc/examples folder of FFMPEG library.
BTW: I also tried setting AVCodecContext->export_side_data to different bit values (0 to FF) understanding that this can indicate the encoder to export the SEI data in every frame to be encoded but no luck.
I appreciate in advance any help from you!
// FfmpegTests.cpp : This file contains the 'main' function. Program execution begins and ends there.
//
#pragma warning(disable : 4996)
extern "C"
{
#include "libavformat/avformat.h"
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavutil/opt.h"
#include "libavutil/avutil.h"
#include "libavutil/error.h"
#include "libavfilter/buffersrc.h"
#include "libavfilter/buffersink.h"
#include "libswscale/swscale.h"
}
#pragma comment(lib, "avcodec.lib")
#pragma comment(lib, "avformat.lib")
#pragma comment(lib, "avfilter.lib")
#pragma comment(lib, "avutil.lib")
#pragma comment(lib, "swscale.lib")
#include <cstdio>
#include <iostream>
#include <chrono>
#include <thread>
static AVFormatContext* fmt_ctx;
static AVCodecContext* dec_ctx;
AVFilterGraph* filter_graph;
AVFilterContext* buffersrc_ctx;
AVFilterContext* buffersink_ctx;
static int video_stream_index = -1;
const char* filter_descr = "scale=78:24,transpose=cclock";
static int64_t last_pts = AV_NOPTS_VALUE;
// FOR SEI NAL INSERTION
const AVOutputFormat* ofmt = NULL;
AVFormatContext* ofmt_ctx = NULL;
int stream_index = 0;
int* stream_mapping = NULL;
int stream_mapping_size = 0;
int FRAMES_COUNT = 0;
const AVCodec* codec_enc;
AVCodecContext* c = NULL;
static int open_input_file(const char* filename)
{
const AVCodec* dec;
int ret;
if ((ret = avformat_open_input(&fmt_ctx, filename, NULL, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open input file\n");
return ret;
}
if ((ret = avformat_find_stream_info(fmt_ctx, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n");
return ret;
}
/* select the video stream */
ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &dec, 0);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find a video stream in the input file\n");
return ret;
}
video_stream_index = ret;
/* create decoding context */
dec_ctx = avcodec_alloc_context3(dec);
if (!dec_ctx)
return AVERROR(ENOMEM);
avcodec_parameters_to_context(dec_ctx, fmt_ctx->streams[video_stream_index]->codecpar);
FRAMES_COUNT = fmt_ctx->streams[video_stream_index]->nb_frames;
/* init the video decoder */
if ((ret = avcodec_open2(dec_ctx, dec, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open video decoder\n");
return ret;
}
return 0;
}
static int init_filters(const char* filters_descr)
{
char args[512];
int ret = 0;
const AVFilter* buffersrc = avfilter_get_by_name("buffer");
const AVFilter* buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut* outputs = avfilter_inout_alloc();
AVFilterInOut* inputs = avfilter_inout_alloc();
AVRational time_base = fmt_ctx->streams[video_stream_index]->time_base;
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE };
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
goto end;
}
/* buffer video source: the decoded frames from the decoder will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer source\n");
goto end;
}
/* buffer video sink: to terminate the filter chain. */
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sink\n");
goto end;
}
ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output pixel format\n");
goto end;
}
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr,
&inputs, &outputs, NULL)) < 0)
goto end;
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
goto end;
end:
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return ret;
}
static void display_frame(const AVFrame* frame, AVRational time_base)
{
int x, y;
uint8_t* p0, * p;
int64_t delay;
if (frame->pts != AV_NOPTS_VALUE) {
if (last_pts != AV_NOPTS_VALUE) {
/* sleep roughly the right amount of time;
* usleep is in microseconds, just like AV_TIME_BASE. */
AVRational timeBaseQ;
timeBaseQ.num = 1;
timeBaseQ.den = AV_TIME_BASE;
delay = av_rescale_q(frame->pts - last_pts, time_base, timeBaseQ);
if (delay > 0 && delay < 1000000)
std::this_thread::sleep_for(std::chrono::microseconds(delay));
}
last_pts = frame->pts;
}
/* Trivial ASCII grayscale display. */
p0 = frame->data[0];
puts("\033c");
for (y = 0; y < frame->height; y++) {
p = p0;
for (x = 0; x < frame->width; x++)
putchar(" .-+#"[*(p++) / 52]);
putchar('\n');
p0 += frame->linesize[0];
}
fflush(stdout);
}
int save_frame_as_jpeg(AVCodecContext* pCodecCtx, AVFrame* pFrame, int FrameNo) {
int ret = 0;
const AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
if (!jpegCodec) {
return -1;
}
AVCodecContext* jpegContext = avcodec_alloc_context3(jpegCodec);
if (!jpegContext) {
return -1;
}
jpegContext->pix_fmt = pCodecCtx->pix_fmt;
jpegContext->height = pFrame->height;
jpegContext->width = pFrame->width;
jpegContext->time_base = AVRational{ 1,10 };
jpegContext->strict_std_compliance = FF_COMPLIANCE_UNOFFICIAL;
ret = avcodec_open2(jpegContext, jpegCodec, NULL);
if (ret < 0) {
return ret;
}
FILE* JPEGFile;
char JPEGFName[256];
AVPacket packet;
packet.data = NULL;
packet.size = 0;
av_init_packet(&packet);
int gotFrame;
ret = avcodec_send_frame(jpegContext, pFrame);
if (ret < 0) {
return ret;
}
ret = avcodec_receive_packet(jpegContext, &packet);
if (ret < 0) {
return ret;
}
sprintf(JPEGFName, "c:\\folder\\dvr-%06d.jpg", FrameNo);
JPEGFile = fopen(JPEGFName, "wb");
fwrite(packet.data, 1, packet.size, JPEGFile);
fclose(JPEGFile);
av_packet_unref(&packet);
avcodec_close(jpegContext);
return 0;
}
int initialize_output_stream(AVFormatContext* input_fctx, const char* out_filename) {
int ret = 0;
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx) {
fprintf(stderr, "Could not create output context\n");
return -1;
}
stream_mapping_size = input_fctx->nb_streams;
stream_mapping = (int*)av_calloc(stream_mapping_size, sizeof(*stream_mapping));
if (!stream_mapping) {
ret = AVERROR(ENOMEM);
return -1;
}
for (int i = 0; i < input_fctx->nb_streams; i++) {
AVStream* out_stream;
AVStream* in_stream = input_fctx->streams[i];
AVCodecParameters* in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO &&
in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
in_codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
stream_mapping[i] = -1;
continue;
}
stream_mapping[i] = stream_index++;
out_stream = avformat_new_stream(ofmt_ctx, NULL);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
return ret;
}
ret = avcodec_parameters_copy(out_stream->codecpar, in_codecpar);
if (ret < 0) {
fprintf(stderr, "Failed to copy codec parameters\n");
return -1;
}
out_stream->codecpar->codec_tag = 0;
}
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", out_filename);
return -1;
}
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
return -1;
}
// ENCODER
codec_enc = avcodec_find_encoder_by_name("libx264");
if (!codec_enc) {
fprintf(stderr, "Codec '%s' not found\n", "libx264");
return -1;
}
c = avcodec_alloc_context3(codec_enc);
if (!c) {
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
c->bit_rate = dec_ctx->bit_rate;
c->width = dec_ctx->width;
c->height = dec_ctx->height;
c->time_base = dec_ctx->time_base;
c->framerate = dec_ctx->framerate;
c->gop_size = dec_ctx->gop_size;
c->max_b_frames = dec_ctx->max_b_frames;
c->pix_fmt = dec_ctx->pix_fmt;
c->time_base = AVRational{ 1,1 };
c->export_side_data = 255;
if (codec_enc->id == AV_CODEC_ID_H264)
av_opt_set(c->priv_data, "preset", "slow", 0);
ret = avcodec_open2(c, codec_enc, NULL);
if (ret < 0) {
fprintf(stderr, "Could not open codec\n");
return ret;
}
}
int add_frame_output_stream(AVFrame* frame) {
int ret;
AVPacket* pkt;
pkt = av_packet_alloc();
ret = avcodec_send_frame(c, frame);
if (ret < 0) {
fprintf(stderr, "Error sending a frame for decoding\n");
return ret;
}
while (ret >= 0) {
ret = avcodec_receive_packet(c, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return 0;
else if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
return -1;
}
pkt->stream_index = stream_mapping[pkt->stream_index];
ret = av_interleaved_write_frame(ofmt_ctx, pkt);
av_packet_unref(pkt);
}
return 0;
}
int main(int argc, char** argv)
{
AVFrame* frame;
AVFrame* filt_frame;
AVPacket* packet;
int ret, count = 0;
// FOR SEI NAL INSERTION
const char* out_filename;
if (argc < 2) {
fprintf(stderr, "Usage: %s file\n", argv[0]);
exit(1);
}
frame = av_frame_alloc();
filt_frame = av_frame_alloc();
packet = av_packet_alloc();
if (!frame || !filt_frame || !packet) {
fprintf(stderr, "Could not allocate frame or packet\n");
exit(1);
}
if ((ret = open_input_file(argv[1])) < 0)
goto end;
if ((ret = init_filters(filter_descr)) < 0)
goto end;
out_filename = argv[2];
initialize_output_stream(fmt_ctx, out_filename);
while (count < FRAMES_COUNT)
{
if ((ret = av_read_frame(fmt_ctx, packet)) < 0)
break;
if (packet->stream_index == video_stream_index) {
ret = avcodec_send_packet(dec_ctx, packet);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while sending a packet to the decoder\n");
break;
}
while (ret >= 0)
{
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
else if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while receiving a frame from the decoder\n");
goto end;
}
frame->pts = frame->best_effort_timestamp;
/* push the decoded frame into the filtergraph */
if (av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while feeding the filtergraph\n");
break;
}
/* pull filtered frames from the filtergraph */
while (1) {
ret = av_buffersink_get_frame(buffersink_ctx, filt_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
if (ret < 0)
goto end;
// display_frame(filt_frame, buffersink_ctx->inputs[0]->time_base);
av_frame_unref(filt_frame);
/* ret = save_frame_as_jpeg(dec_ctx, frame, dec_ctx->frame_number);
if (ret < 0)
goto end; */
//2. Add metadata to frames SEI
ret = av_frame_make_writable(frame);
if (ret < 0)
exit(1);
char sideDataSei[43] = "139FB1A9446A4DEC8CBF65B1E12D2CFDHola";
const char* sideDataMsg = "139FB1A9446A4DEC8CBF65B1E12D2CFDHola";
size_t sideDataSize = sizeof(sideDataMsg);
AVBufferRef* sideDataBuffer = av_buffer_alloc(20);
sideDataBuffer->data = (uint8_t*)sideDataMsg;
AVFrameSideData* sideData = av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_SEI_UNREGISTERED, sideDataBuffer);
ret = add_frame_output_stream(frame);
if (ret < 0)
goto end;
}
av_frame_unref(frame);
count++;
}
}
av_packet_unref(packet);
}
av_write_trailer(ofmt_ctx);
end:
avfilter_graph_free(&filter_graph);
avcodec_free_context(&dec_ctx);
avformat_close_input(&fmt_ctx);
av_frame_free(&frame);
av_frame_free(&filt_frame);
av_packet_free(&packet);
if (ret < 0 && ret != AVERROR_EOF) {
char errBuf[AV_ERROR_MAX_STRING_SIZE]{ 0 };
int res = av_strerror(ret, errBuf, AV_ERROR_MAX_STRING_SIZE);
fprintf(stderr, "Error: %s\n", errBuf);
exit(1);
}
exit(0);
}
Well, I came up with this solution from a friend, just had to add:
*av_opt_set_int(c->priv_data, "udu_sei", 1, 0);*
In the function initialize_output_stream after all parameters are set for AVCodecContext (c) that is being used for the output stream encoding.
Hope this helps someone!

Capture and encode desktop with libav in real time not giving corect images

As part of a larger project I want to be able to capture and encode the desktop frame by frame in real time. I have the following test code to reproduce the issue shown in the screenshot:
#include <stdlib.h>
#include <stdio.h>
#include <iostream>
#include <fstream>
#include <string>
#include <string.h>
#include <math.h>
extern "C"
{
#include "libavdevice/avdevice.h"
#include "libavutil/channel_layout.h"
#include "libavutil/mathematics.h"
#include "libavutil/opt.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
}
/* 5 seconds stream duration */
#define STREAM_DURATION 5.0
#define STREAM_FRAME_RATE 25 /* 25 images/s */
#define STREAM_NB_FRAMES ((int)(STREAM_DURATION * STREAM_FRAME_RATE))
#define STREAM_PIX_FMT AV_PIX_FMT_YUV420P /* default pix_fmt */
int videoStreamIndx;
int framerate = 30;
int width = 1920;
int height = 1080;
int encPacketCounter;
AVFormatContext* ifmtCtx;
AVCodecContext* avcodecContx;
AVFormatContext* ofmtCtx;
AVStream* videoStream;
AVCodecContext* avCntxOut;
AVPacket* avPkt;
AVFrame* avFrame;
AVFrame* outFrame;
SwsContext* swsCtx;
std::ofstream fs;
AVDictionary* ConfigureScreenCapture()
{
AVDictionary* options = NULL;
//Try adding "-rtbufsize 100M" as in https://stackoverflow.com/questions/6766333/capture-windows-screen-with-ffmpeg
av_dict_set(&options, "rtbufsize", "100M", 0);
av_dict_set(&options, "framerate", std::to_string(framerate).c_str(), 0);
char buffer[16];
sprintf(buffer, "%dx%d", width, height);
av_dict_set(&options, "video_size", buffer, 0);
return options;
}
AVCodecParameters* ConfigureAvCodec()
{
AVCodecParameters* av_codec_par_out = avcodec_parameters_alloc();
av_codec_par_out->width = width;
av_codec_par_out->height = height;
av_codec_par_out->bit_rate = 40000;
av_codec_par_out->codec_id = AV_CODEC_ID_H264; //AV_CODEC_ID_MPEG4; //Try H.264 instead of MPEG4
av_codec_par_out->codec_type = AVMEDIA_TYPE_VIDEO;
av_codec_par_out->format = 0;
return av_codec_par_out;
}
int GetVideoStreamIndex()
{
int VideoStreamIndx = -1;
avformat_find_stream_info(ifmtCtx, NULL);
/* find the first video stream index . Also there is an API available to do the below operations */
for (int i = 0; i < (int)ifmtCtx->nb_streams; i++) // find video stream position/index.
{
if (ifmtCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
{
VideoStreamIndx = i;
break;
}
}
if (VideoStreamIndx == -1)
{
}
return VideoStreamIndx;
}
void CreateFrames(AVCodecParameters* av_codec_par_in, AVCodecParameters* av_codec_par_out)
{
avFrame = av_frame_alloc();
avFrame->width = avcodecContx->width;
avFrame->height = avcodecContx->height;
avFrame->format = av_codec_par_in->format;
av_frame_get_buffer(avFrame, 0);
outFrame = av_frame_alloc();
outFrame->width = avCntxOut->width;
outFrame->height = avCntxOut->height;
outFrame->format = av_codec_par_out->format;
av_frame_get_buffer(outFrame, 0);
}
bool Init()
{
AVCodecParameters* avCodecParOut = ConfigureAvCodec();
AVDictionary* options = ConfigureScreenCapture();
AVInputFormat* ifmt = av_find_input_format("gdigrab");
auto ifmtCtxLocal = avformat_alloc_context();
if (avformat_open_input(&ifmtCtxLocal, "desktop", ifmt, &options) < 0)
{
return false;
}
ifmtCtx = ifmtCtxLocal;
videoStreamIndx = GetVideoStreamIndex();
AVCodecParameters* avCodecParIn = avcodec_parameters_alloc();
avCodecParIn = ifmtCtx->streams[videoStreamIndx]->codecpar;
AVCodec* avCodec = avcodec_find_decoder(avCodecParIn->codec_id);
if (avCodec == NULL)
{
return false;
}
avcodecContx = avcodec_alloc_context3(avCodec);
if (avcodec_parameters_to_context(avcodecContx, avCodecParIn) < 0)
{
return false;
}
//av_dict_set
int value = avcodec_open2(avcodecContx, avCodec, NULL); //Initialize the AVCodecContext to use the given AVCodec.
if (value < 0)
{
return false;
}
AVOutputFormat* ofmt = av_guess_format("h264", NULL, NULL);
if (ofmt == NULL)
{
return false;
}
auto ofmtCtxLocal = avformat_alloc_context();
avformat_alloc_output_context2(&ofmtCtxLocal, ofmt, NULL, NULL);
if (ofmtCtxLocal == NULL)
{
return false;
}
ofmtCtx = ofmtCtxLocal;
AVCodec* avCodecOut = avcodec_find_encoder(avCodecParOut->codec_id);
if (avCodecOut == NULL)
{
return false;
}
videoStream = avformat_new_stream(ofmtCtx, avCodecOut);
if (videoStream == NULL)
{
return false;
}
avCntxOut = avcodec_alloc_context3(avCodecOut);
if (avCntxOut == NULL)
{
return false;
}
if (avcodec_parameters_copy(videoStream->codecpar, avCodecParOut) < 0)
{
return false;
}
if (avcodec_parameters_to_context(avCntxOut, avCodecParOut) < 0)
{
return false;
}
avCntxOut->gop_size = 30; //3; //Use I-Frame frame every 30 frames.
avCntxOut->max_b_frames = 0;
avCntxOut->time_base.num = 1;
avCntxOut->time_base.den = framerate;
//avio_open(&ofmtCtx->pb, "", AVIO_FLAG_READ_WRITE);
if (avformat_write_header(ofmtCtx, NULL) < 0)
{
return false;
}
value = avcodec_open2(avCntxOut, avCodecOut, NULL); //Initialize the AVCodecContext to use the given AVCodec.
if (value < 0)
{
return false;
}
if (avcodecContx->codec_id == AV_CODEC_ID_H264)
{
av_opt_set(avCntxOut->priv_data, "preset", "ultrafast", 0);
av_opt_set(avCntxOut->priv_data, "zerolatency", "1", 0);
av_opt_set(avCntxOut->priv_data, "tune", "ull", 0);
}
if ((ofmtCtx->oformat->flags & AVFMT_GLOBALHEADER) != 0)
{
avCntxOut->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
CreateFrames(avCodecParIn, avCodecParOut);
swsCtx = sws_alloc_context();
if (sws_init_context(swsCtx, NULL, NULL) < 0)
{
return false;
}
swsCtx = sws_getContext(avcodecContx->width, avcodecContx->height, avcodecContx->pix_fmt,
avCntxOut->width, avCntxOut->height, avCntxOut->pix_fmt, SWS_FAST_BILINEAR,
NULL, NULL, NULL);
if (swsCtx == NULL)
{
return false;
}
return true;
}
void Encode(AVCodecContext* enc_ctx, AVFrame* frame, AVPacket* pkt)
{
int ret;
/* send the frame to the encoder */
ret = avcodec_send_frame(enc_ctx, frame);
if (ret < 0)
{
return;
}
while (ret >= 0)
{
ret = avcodec_receive_packet(enc_ctx, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return;
if (ret < 0)
{
return;
}
fs.write((char*)pkt->data, pkt->size);
av_packet_unref(pkt);
}
}
void EncodeFrames(int noFrames)
{
int frameCount = 0;
avPkt = av_packet_alloc();
AVPacket* outPacket = av_packet_alloc();
encPacketCounter = 0;
while (av_read_frame(ifmtCtx, avPkt) >= 0)
{
if (frameCount++ == noFrames)
break;
if (avPkt->stream_index != videoStreamIndx) continue;
avcodec_send_packet(avcodecContx, avPkt);
if (avcodec_receive_frame(avcodecContx, avFrame) >= 0) // Frame successfully decoded :)
{
outPacket->data = NULL; // packet data will be allocated by the encoder
outPacket->size = 0;
outPacket->pts = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
if (outPacket->dts != AV_NOPTS_VALUE)
outPacket->dts = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
outPacket->dts = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
outPacket->duration = av_rescale_q(1, avCntxOut->time_base, videoStream->time_base);
outFrame->pts = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
outFrame->pkt_duration = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
encPacketCounter++;
int sts = sws_scale(swsCtx,
avFrame->data, avFrame->linesize, 0, avFrame->height,
outFrame->data, outFrame->linesize);
/* make sure the frame data is writable */
auto ret = av_frame_make_writable(outFrame);
if (ret < 0)
break;
Encode(avCntxOut, outFrame, outPacket);
}
av_frame_unref(avFrame);
av_packet_unref(avPkt);
}
}
void Dispose()
{
fs.close();
auto ifmtCtxLocal = ifmtCtx;
avformat_close_input(&ifmtCtx);
avformat_free_context(ifmtCtx);
avcodec_free_context(&avcodecContx);
}
int main(int argc, char** argv)
{
avdevice_register_all();
fs.open("out.h264");
if (Init())
{
EncodeFrames(300);
}
else
{
std::cout << "Failed to Init \n";
}
Dispose();
return 0;
}
As far as I can tell the setup of the encoding process is correct as it is largely unchanged from how the example given in the official documentation is working: https://libav.org/documentation/doxygen/master/encode__video_8c_source.html
However there is limited documentation around the desktop capture online so I am not sure if I have set that up correctly.
We have to open the out.h264 as binary file.
Replace fs.open("out.h264"); with fs.open("out.h264", std::ios::binary);.
The default file type in Windows is "text file".
That means that each \n in converted to \r\n when writing, and the encoded stream get "messed up".
It took me quite a long time to figure out the problem...
There is another small issue:
There is a missing loop at the end, that flushes the remaining encoded packets.
We can use FFprobe for counting the number of encoded frames:
ffprobe -v error -select_streams v:0 -count_frames -show_entries stream=nb_read_frames -print_format csv out.h264
The result is 263 instead of 300.
The solution is adding the following loop at the end of void EncodeFrames(int noFrames) function:
int ret = 0;
avcodec_send_frame(avCntxOut, NULL);
do
{
av_packet_unref(outPacket);
ret = avcodec_receive_packet(avCntxOut, outPacket);
if (!ret)
{
fs.write((char*)outPacket->data, outPacket->size);
}
} while (!ret);
This is not a solution to the problem directly but maybe so?
AVDictionary * pDic = NULL;
av_dict_set(&pDic, "tune", "zerolatency", 0);
avcodec_open2(avCntxOut, avCodecOut, &pDic);

FFMpeg: extracting audio frames with avcodec creates noise

In my Android app, I implemented the FFMpeg library, and try to use it so I can extract audio samples from an audio file on the fly.
Here is what I did (I simplified the code here so it's easier to read):
AVPacket packet;
AVCodecContext *codecContext = NULL;
AVFormatContext *formatContext;
AVFrame *frame = NULL;
SwrContext *swrContext;
int audio_stream_index = -1;
int ret;
uint8_t *localBuffer;
int FFMpegPlayer::createFFmpeg(const char *filename)
{
int ret;
AVCodec *dec;
frame = av_frame_alloc();
av_register_all();
avformat_open_input(&formatContext, filename, NULL, NULL))
avformat_find_stream_info(formatContext, NULL))
// select the audio stream
audio_stream_index = av_find_best_stream(formatContext, AVMEDIA_TYPE_AUDIO, -1, -1, &dec, 0);
// create decoding context
codecContext = avcodec_alloc_context3(dec);
avcodec_parameters_to_context(codecContext, formatContext->streams[audio_stream_index]->codecpar);
av_opt_set_int(codecContext, "refcounted_frames", 1, 0);
// init the audio decoder
avcodec_open2(codecContext, dec, NULL))
swrContext = swr_alloc();
// we assume here that the audio file is a 44100 Hz stereo audio file
localBuffer = (uint8_t *) av_malloc(44100 * 2);
swr_alloc_set_opts(swrContext, AV_CH_LAYOUT_STEREO, AV_SAMPLE_FMT_FLT, codecContext->sample_rate,
codecContext->channel_layout, codecContext->sample_fmt, codecContext->sample_rate, 0,
NULL);
swr_init(swrContext);
return 0;
}
void FFMpegPlayer::getPcmFloat(float *buffer)
{
// init :
int i, ch, dataSize;
bool extraxted = false;
float sample = 0;
// extract :
while (av_read_frame(formatContext, &packet) >= 0 && !extraxted)
{
if (packet.stream_index == audio_stream_index)
{
// send the packet with the compressed data to the decoder
ret = avcodec_send_packet(codecContext, &packet);
// read all the output frames (in general there may be any number of them
while (ret >= 0)
{
ret = avcodec_receive_frame(codecContext, frame);
if (ret == AVERROR(EAGAIN))
{
LOGW("AVERROR(EAGAIN)\n");
break;
}
else if (ret == AVERROR_EOF)
{
LOGW("AVERROR_EOF\n");
break;
}
dataSize = av_get_bytes_per_sample(codecContext->sample_fmt);
swr_convert(swrContext, &localBuffer, 44100 * 2, (const uint8_t **) frame->data, frame->nb_samples);
int a = 0;
for (i = 0; i < frame->nb_samples; i++)
{
for (ch = 0; ch < codecContext->channels; ch++)
{
memcpy(&sample, &localBuffer[(codecContext->channels * i + ch) * dataSize], dataSize);
buffer[a] = sample;
a++;
}
}
// exit extract:
extraxted = true;
}
}
}
}
Anytime I need audio samples, I call the getPcmFloat() function.
Thanks to that code, I can clearly listen to the audio file.
The problem is: I have some crackling in the sound, and I have no idea where it comes from, or how to fix it.
Does anyone know how to get the exact frames whithout glitches?
Thanks for your help.

FFMPEG H264 encode each single image

i encode currently a QImage from RGB888 to H264, but i want to encode each image (even if this is not the perfect way) by itself.
Im able to encode the image, but its needed to send the same image 46 times. And i dont know what i do wrong (probably wrong config of the encode, but i cannot find the issue there).
Afterwards i decode this image and then convert it back to a QImage. I do this only for testing some other code.
avcodec_register_all();
AVCodec *nVidiaCodec = avcodec_find_encoder_by_name("h264_nvenc");
if (!nVidiaCodec)
{
return false;
}
AVCodecContext* av_codec_context_ = NULL;
av_codec_context_ = avcodec_alloc_context3(nVidiaCodec);
if (!av_codec_context_)
{
return false;
}
av_codec_context_->width = dst->width;
av_codec_context_->height = dst->height;
av_codec_context_->pix_fmt = AV_PIX_FMT_YUV420P;
av_codec_context_->gop_size = 1;
av_codec_context_->keyint_min = 0;
av_codec_context_->scenechange_threshold = 0;
av_codec_context_->bit_rate = 8000000;
av_codec_context_->time_base.den = 1;
av_codec_context_->time_base.num = 1;
av_codec_context_->refs = 0;
av_codec_context_->qmin = 1;
av_codec_context_->qmax = 1;
av_codec_context_->b_frame_strategy = 0;
av_codec_context_->max_b_frames = 0;
av_codec_context_->thread_count = 1;
av_opt_set(av_codec_context_, "preset", "slow", 0);
av_opt_set(av_codec_context_, "tune", "zerolatency", 0);
int ret = avcodec_open2(av_codec_context_, nVidiaCodec, NULL);
if (0 > ret)
{
return false;
}
AVFrame *picture = av_frame_alloc();
picture->format = AV_PIX_FMT_RGB24;
picture->width = dst->width;
picture->height = dst->height;
ret = avpicture_fill((AVPicture *)picture, imgSrc.bits(), AV_PIX_FMT_RGB24, dst->width, dst->height);
if (0 > ret)
{
return false;
}
AVFrame *tmp_picture = av_frame_alloc();
tmp_picture->format = AV_PIX_FMT_YUV420P;
tmp_picture->width = dst->width;
tmp_picture->height = dst->height;
ret = av_frame_get_buffer(tmp_picture, 32);
SwsContext *img_convert_ctx = sws_getContext(av_codec_context_->width, av_codec_context_->height, AV_PIX_FMT_RGB24, av_codec_context_->width, av_codec_context_->height, av_codec_context_->pix_fmt, SWS_BICUBIC, NULL, NULL, NULL);
if (!img_convert_ctx)
{
return false;
}
ret = sws_scale(img_convert_ctx, picture->data, picture->linesize, 0, av_codec_context_->height, tmp_picture->data, tmp_picture->linesize);
if (0 > ret)
{
return false;
}
ret = avcodec_send_frame(av_codec_context_, tmp_picture);
if (0 > ret)
{
return false;
}
AVPacket pkt;
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
do
{
ret = avcodec_receive_packet(av_codec_context_, &pkt);
if (ret == 0)
{
break;
}
else if ((ret < 0) && (ret != AVERROR(EAGAIN)))
{
return false;
}
else if (ret == AVERROR(EAGAIN))
{
ret = avcodec_send_frame(av_codec_context_, tmp_picture);
if (0 > ret)
{
return false;
}
}
} while (ret == 0);
// the do while is called 46 times, then i get the packet, but i want to get the packet at the first call
It would be very nice if you can help me.
Thanks guys.
I assume you just want to encode a single frame. You need to flush the encoder after you have sent your single uncompressed frame by sending NULL instead of a valid buffer.
int result = 0;
// encoder init
// send one uncompressed frame
result = avcodec_send_frame(av_codec_context_, tmp_picture);
if (result < 0) return false;
// send NULL to indicate flushing
result = avcodec_send_frame(av_codec_context_, NULL);
if (result < 0) return false;
while (result != AVERROR_EOF)
{
result = avcodec_receive_packet(av_codec_context_, &pkt);
if (!result)
{
// you should have your encoded frame; do something with it
}
}

FFMPEG. Read frame, process it, put it to output video. Copy sound stream unchanged

I want to apply processing to a video clip with sound track, extract and process frame by frame and write result to output file. Number of frames, size of frame and speed remains unchanged in output clip. Also I want to keep the same audio track as I have in source.
I can read clip, decode frames and process then using opencv. Audio packets are also writes fine. I'm stuck on forming output video stream.
The minimal runnable code I have for now (sorry it not so short, but cant do it shorter):
extern "C" {
#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
#include "libavcodec/avcodec.h"
#include <libavutil/opt.h>
#include <libavdevice/avdevice.h>
#include <libswscale/swscale.h>
}
#include "opencv2/opencv.hpp"
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1)
#define av_frame_alloc avcodec_alloc_frame
#endif
using namespace std;
using namespace cv;
static void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt, const char *tag)
{
AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
char buf1[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_string(buf1, pkt->pts);
char buf2[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_string(buf1, pkt->dts);
char buf3[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_string(buf1, pkt->duration);
char buf4[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_time_string(buf1, pkt->pts, time_base);
char buf5[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_time_string(buf1, pkt->dts, time_base);
char buf6[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_time_string(buf1, pkt->duration, time_base);
printf("pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
buf1, buf4,
buf2, buf5,
buf3, buf6,
pkt->stream_index);
}
int main(int argc, char **argv)
{
AVOutputFormat *ofmt = NULL;
AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
AVPacket pkt;
AVFrame *pFrame = NULL;
AVFrame *pFrameRGB = NULL;
int frameFinished = 0;
pFrame = av_frame_alloc();
pFrameRGB = av_frame_alloc();
const char *in_filename, *out_filename;
int ret, i;
in_filename = "../../TestClips/Audio Video Sync Test.mp4";
out_filename = "out.mp4";
// Initialize FFMPEG
av_register_all();
// Get input file format context
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0)
{
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
// Extract streams description
if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0)
{
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
// Print detailed information about the input or output format,
// such as duration, bitrate, streams, container, programs, metadata, side data, codec and time base.
av_dump_format(ifmt_ctx, 0, in_filename, 0);
// Allocate an AVFormatContext for an output format.
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx)
{
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
// The output container format.
ofmt = ofmt_ctx->oformat;
// Allocating output streams
for (i = 0; i < ifmt_ctx->nb_streams; i++)
{
AVStream *in_stream = ifmt_ctx->streams[i];
AVStream *out_stream = avformat_new_stream(ofmt_ctx, in_stream->codec->codec);
if (!out_stream)
{
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_copy_context(out_stream->codec, in_stream->codec);
if (ret < 0)
{
fprintf(stderr, "Failed to copy context from input to output stream codec context\n");
goto end;
}
out_stream->codec->codec_tag = 0;
if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
{
out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
}
// Show output format info
av_dump_format(ofmt_ctx, 0, out_filename, 1);
// Open output file
if (!(ofmt->flags & AVFMT_NOFILE))
{
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0)
{
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
// Write output file header
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0)
{
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
// Search for input video codec info
AVCodec *in_codec = nullptr;
AVCodecContext* avctx = nullptr;
int video_stream_index = -1;
for (int i = 0; i < ifmt_ctx->nb_streams; i++)
{
if (ifmt_ctx->streams[i]->codec->coder_type == AVMEDIA_TYPE_VIDEO)
{
video_stream_index = i;
avctx = ifmt_ctx->streams[i]->codec;
in_codec = avcodec_find_decoder(avctx->codec_id);
if (!in_codec)
{
fprintf(stderr, "in codec not found\n");
exit(1);
}
break;
}
}
// Search for output video codec info
AVCodec *out_codec = nullptr;
AVCodecContext* o_avctx = nullptr;
int o_video_stream_index = -1;
for (int i = 0; i < ofmt_ctx->nb_streams; i++)
{
if (ofmt_ctx->streams[i]->codec->coder_type == AVMEDIA_TYPE_VIDEO)
{
o_video_stream_index = i;
o_avctx = ofmt_ctx->streams[i]->codec;
out_codec = avcodec_find_encoder(o_avctx->codec_id);
if (!out_codec)
{
fprintf(stderr, "out codec not found\n");
exit(1);
}
break;
}
}
// openCV pixel format
AVPixelFormat pFormat = AV_PIX_FMT_RGB24;
// Data size
int numBytes = avpicture_get_size(pFormat, avctx->width, avctx->height);
// allocate buffer
uint8_t *buffer = (uint8_t *)av_malloc(numBytes * sizeof(uint8_t));
// fill frame structure
avpicture_fill((AVPicture *)pFrameRGB, buffer, pFormat, avctx->width, avctx->height);
// frame area
int y_size = avctx->width * avctx->height;
// Open input codec
avcodec_open2(avctx, in_codec, NULL);
// Main loop
while (1)
{
AVStream *in_stream, *out_stream;
ret = av_read_frame(ifmt_ctx, &pkt);
if (ret < 0)
{
break;
}
in_stream = ifmt_ctx->streams[pkt.stream_index];
out_stream = ofmt_ctx->streams[pkt.stream_index];
log_packet(ifmt_ctx, &pkt, "in");
// copy packet
pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base, AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
log_packet(ofmt_ctx, &pkt, "out");
if (pkt.stream_index == video_stream_index)
{
avcodec_decode_video2(avctx, pFrame, &frameFinished, &pkt);
if (frameFinished)
{
struct SwsContext *img_convert_ctx;
img_convert_ctx = sws_getCachedContext(NULL,
avctx->width,
avctx->height,
avctx->pix_fmt,
avctx->width,
avctx->height,
AV_PIX_FMT_BGR24,
SWS_BICUBIC,
NULL,
NULL,
NULL);
sws_scale(img_convert_ctx,
((AVPicture*)pFrame)->data,
((AVPicture*)pFrame)->linesize,
0,
avctx->height,
((AVPicture *)pFrameRGB)->data,
((AVPicture *)pFrameRGB)->linesize);
sws_freeContext(img_convert_ctx);
// Do some image processing
cv::Mat img(pFrame->height, pFrame->width, CV_8UC3, pFrameRGB->data[0],false);
cv::GaussianBlur(img,img,Size(5,5),3);
cv::imshow("Display", img);
cv::waitKey(5);
// --------------------------------
// Transform back to initial format
// --------------------------------
img_convert_ctx = sws_getCachedContext(NULL,
avctx->width,
avctx->height,
AV_PIX_FMT_BGR24,
avctx->width,
avctx->height,
avctx->pix_fmt,
SWS_BICUBIC,
NULL,
NULL,
NULL);
sws_scale(img_convert_ctx,
((AVPicture*)pFrameRGB)->data,
((AVPicture*)pFrameRGB)->linesize,
0,
avctx->height,
((AVPicture *)pFrame)->data,
((AVPicture *)pFrame)->linesize);
// --------------------------------------------
// Something must be here
// --------------------------------------------
//
// Write fideo frame (How to write frame to output stream ?)
//
// --------------------------------------------
sws_freeContext(img_convert_ctx);
}
}
else // write sound frame
{
ret = av_interleaved_write_frame(ofmt_ctx, &pkt);
}
if (ret < 0)
{
fprintf(stderr, "Error muxing packet\n");
break;
}
// Decrease packet ref counter
av_packet_unref(&pkt);
}
av_write_trailer(ofmt_ctx);
end:
avformat_close_input(&ifmt_ctx);
// close output
if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE))
{
avio_closep(&ofmt_ctx->pb);
}
avformat_free_context(ofmt_ctx);
if (ret < 0 && ret != AVERROR_EOF)
{
char buf_err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
av_make_error_string(buf_err, AV_ERROR_MAX_STRING_SIZE, ret);
fprintf(stderr, "Error occurred: %s\n", buf_err);
return 1;
}
avcodec_close(avctx);
av_free(pFrame);
av_free(pFrameRGB);
return 0;
}
Your original code segfaults in my case. Initializing the output codec context seems to fix it. The code below works for me but I didn't test the OpenCV stuff as I don't have the lib installed.
Get the codec context:
// Search for output video codec info
AVCodec *out_codec = NULL;
AVCodecContext* o_avctx = NULL;
int o_video_stream_index = -1;
for (int i = 0; i < ofmt_ctx->nb_streams; i++)
{
if (ofmt_ctx->streams[i]->codec->coder_type == AVMEDIA_TYPE_VIDEO)
{
o_video_stream_index = i;
out_codec = avcodec_find_encoder(ofmt_ctx->streams[i]->codec->codec_id);
o_avctx = avcodec_alloc_context3(out_codec);
o_avctx->height = avctx->height;
o_avctx->width = avctx->width;
o_avctx->sample_aspect_ratio = avctx->sample_aspect_ratio;
if (out_codec->pix_fmts)
o_avctx->pix_fmt = out_codec->pix_fmts[0];
else
o_avctx->pix_fmt = avctx->pix_fmt;
o_avctx->time_base = avctx->time_base;
avcodec_open2(o_avctx, out_codec, NULL);
}
}
Encode and write:
// Main loop
while (1)
{
...
if (pkt.stream_index == video_stream_index)
{
avcodec_decode_video2(avctx, pFrame, &frameFinished, &pkt);
if (frameFinished)
{
...
// --------------------------------------------
// Something must be here
// --------------------------------------------
int got_packet = 0;
AVPacket enc_pkt = { 0 };
av_init_packet(&enc_pkt);
avcodec_encode_video2(o_avctx, &enc_pkt, pFrame, &got_packet);
av_interleaved_write_frame(ofmt_ctx, &enc_pkt);
....
}
}
you should assign processed frame's packets information to your Original packets then pass it to av_interleaved_write_frame