Related
I am trying to create a real-time stream of my screen using ffmpeg and encoder h264_nvenc but there's 5-6 frames delay between my mouse move and mouse move on the translation(first 5-6 frames are black)
I need to somehow encode 1 image of my screen using h264_nvenc and receive it as valid frame(packet) or maybe decrease delay to 1-2 frames
Maybe there are some other methods to get the image right away
UPD:
I realised that my latency is because of encoder and decoder frames delay so is there any way to completely remove this frame delays?(Added decoder init function and decode function)
I've done some tests and managed to receive image right away with decoder and encoder flush method but cant do this the second time
Encode function:
static void encode(AVCodecContext* enc_ctx, AVFrame* frame, AVPacket* pkt, AVPacket** new_packet)
{
int ret;
ret = avcodec_send_frame(enc_ctx, frame);
if (ret < 0) {
fprintf(stderr, "Error sending a frame for encoding\n");
exit(1);
}
int size = 0;
while (ret >= 0) {
ret = avcodec_receive_packet(enc_ctx, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
{
//printf("count: %i size: %i\n", count, size);
return;
}
else if (ret < 0) {
fprintf(stderr, "Error during encoding\n");
exit(1);
}
size += pkt->size;
*new_packet = av_packet_clone(pkt);
av_packet_unref(pkt);
}
}
FFMpeg encoder initialization:
void InitializeFFmpegEncoder()
{
const AVCodec* codec;
int i, ret, x, y;
FILE* f;
// Encoder
codec = avcodec_find_encoder_by_name("h264_nvenc"); // hevc_nvenc h264_nvenc
if (!codec) {
fprintf(stderr, "Codec '%s' not found\n", "hevc");
exit(1);
}
/*codec = avcodec_find_encoder(AV_CODEC_ID_H265);
if (!codec) {
fprintf(stderr, "Codec '%s' not found\n", "hevc");
exit(1);
}*/
cout << "Encoder codec name: " << codec->name << endl;
cout << "1" << endl;
enc_c = avcodec_alloc_context3(codec);
if (!enc_c) {
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
cout << "2" << endl;
pkt = av_packet_alloc();
if (!pkt)
exit(1);
cout << "3" << endl;
/* put sample parameters */
enc_c->bit_rate = 192000000;
/* resolution must be a multiple of two */
enc_c->width = 1920;
enc_c->height = 1080;
/* frames per second */
enc_c->time_base = AVRational(1, 1);
//enc_c->framerate = AVRational(60, 1);
/* emit one intra frame every ten frames
* check frame pict_type before passing frame
* to encoder, if frame->pict_type is AV_PICTURE_TYPE_I
* then gop_size is ignored and the output of encoder
* will always be I frame irrespective to gop_size
*/
enc_c->gop_size = 1;
enc_c->max_b_frames = 0;
enc_c->pix_fmt = AV_PIX_FMT_BGR0;
enc_c->keyint_min = 0;
if (codec->id == AV_CODEC_ID_H264)
{
//av_opt_set(enc_c->priv_data, "preset", "p1", 0);
//av_opt_set(enc_c->priv_data, "tune", "ull", 0);
//av_opt_set(enc_c->priv_data, "zerolatency", "1", 0);
//av_opt_set(enc_c->priv_data, "preset", "p1", 0);
//av_opt_set(enc_c->priv_data, "tune", "ull", 0);
//av_opt_set(enc_c->priv_data, "strict_gop", "1", 0);
//av_opt_set(enc_c->priv_data, "preset", "lossless", 0);
//av_opt_set(enc_c->priv_data, "zerolatency", "1", 0);
}
cout << "4" << endl;
/* open it */
ret = avcodec_open2(enc_c, codec, NULL);
if (ret < 0) {
printf("Could not open codec: %s\n", av_make_error_string((char[64])(0), 64, ret));
exit(1);
}
cout << "5" << endl;
frame = av_frame_alloc();
if (!frame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
frame->format = AV_PIX_FMT_BGR0; // AV_PIX_FMT_YUV444P AV_PIX_FMT_ARGB
frame->width = enc_c->width;
frame->height = enc_c->height;
cout << "6" << endl;
ret = av_frame_get_buffer(frame, 0);
if (ret < 0) {
fprintf(stderr, "Could not allocate the video frame data\n");
exit(1);
}
cout << "7" << endl;
}
Decoder init function:
int InitializeFFmpegDecoder()
{
int ret;
const AVCodec* decoder = NULL;
enum AVHWDeviceType type;
int i;
decoder = avcodec_find_decoder_by_name("h264_cuvid"); // h264_cuvid hevc_cuvid
if (!decoder) {
fprintf(stderr, "Codec not found\n");
exit(1);
}
cout << "Decoder name: " << decoder->name << endl;
if (!(decoder_ctx = avcodec_alloc_context3(decoder)))
return AVERROR(ENOMEM);
decoder_ctx->get_format = ffmpeg_GetFormat;
decoder_ctx->gop_size = 0;
decoder_ctx->max_b_frames = 0;
decoder_ctx->keyint_min = 0;
decoder_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
//decoder_ctx->get_format = get_format;
if ((ret = avcodec_open2(decoder_ctx, decoder, NULL)) < 0) {
fprintf(stderr, "Failed to open codec for stream \n");
return -1;
}
if (ctx == NULL)
{
ctx = sws_getContext(1920, 1080,
AV_PIX_FMT_NV12, 1920, 1080, // AV_PIX_FMT_YUV420P AV_PIX_FMT_NV12
AV_PIX_FMT_BGR0, 0, 0, 0, 0);
}
}
Decode function:
static void decode(AVCodecContext* dec_ctx, AVFrame* frame, AVPacket* pkt, char* bgra_image)
{
int ret;
ret = avcodec_send_packet(dec_ctx, pkt);
if (ret < 0) {
fprintf(stderr, "Error sending a packet for decoding\n");
exit(1);
}
while (ret >= 0) {
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return;
else if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
exit(1);
}
char* outData[1] = { bgra_image }; // RGB24 have one plane
int outLinesize[1] = { 4 * 1920 }; // RGB stride
sws_scale(ctx, frame->data, frame->linesize, 0, 1080, (uint8_t* const*)outData, outLinesize);
//*new_frame = av_frame_clone(frame);
//av_frame_ref()
//break;
}
}
You can check av_seek_frame(av_format_ctx, video_stream_index, timestamp, AVSEEK_FLAG_BACKWARD) out for your decoder. Go to the frame you want to encode then encode that frame.
Encoding a single frame works just like in the loop after you seek to the frame you want to seek, note that av_seek seeks iFrames.
For h264_nvenc set option delay to 0
But for decoder h264_cuvid I don't know how to make 0 delay stream so I switched to DXVA2 and set max_b_frames to 0
I have a C++ project which creates 7/24 WebTV like RTMP stream and allows operations like changing current content on runtime, seeking content, looping through a playlist which is constructed by a json array, also supports changing whole playlist on runtime.
Currently i am reading H264 and AAC encoded packets from mp4 files then sending them to destination RTMP server after adjusting their PTS & DTS values without any encoding or decoding.
But i want to apply overlay images to raw frames using FFmpeg "overlay" filter after decoding H264 packets. I looked at sample which came with FFmpeg examples ;
#define _XOPEN_SOURCE 600 /* for usleep */
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavutil/opt.h>
const char *filter_descr = "scale=78:24,transpose=cclock";
/* other way:
scale=78:24 [scl]; [scl] transpose=cclock // assumes "[in]" and "[out]" to be input output pads respectively
*/
static AVFormatContext *fmt_ctx;
static AVCodecContext *dec_ctx;
AVFilterContext *buffersink_ctx;
AVFilterContext *buffersrc_ctx;
AVFilterGraph *filter_graph;
static int video_stream_index = -1;
static int64_t last_pts = AV_NOPTS_VALUE;
static int open_input_file(const char *filename)
{
int ret;
AVCodec *dec;
if ((ret = avformat_open_input(&fmt_ctx, filename, NULL, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open input file\n");
return ret;
}
if ((ret = avformat_find_stream_info(fmt_ctx, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n");
return ret;
}
/* select the video stream */
ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &dec, 0);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find a video stream in the input file\n");
return ret;
}
video_stream_index = ret;
/* create decoding context */
dec_ctx = avcodec_alloc_context3(dec);
if (!dec_ctx)
return AVERROR(ENOMEM);
avcodec_parameters_to_context(dec_ctx, fmt_ctx->streams[video_stream_index]->codecpar);
/* init the video decoder */
if ((ret = avcodec_open2(dec_ctx, dec, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open video decoder\n");
return ret;
}
return 0;
}
static int init_filters(const char *filters_descr)
{
char args[512];
int ret = 0;
const AVFilter *buffersrc = avfilter_get_by_name("buffer");
const AVFilter *buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut *outputs = avfilter_inout_alloc();
AVFilterInOut *inputs = avfilter_inout_alloc();
AVRational time_base = fmt_ctx->streams[video_stream_index]->time_base;
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE };
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
goto end;
}
/* buffer video source: the decoded frames from the decoder will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer source\n");
goto end;
}
/* buffer video sink: to terminate the filter chain. */
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sink\n");
goto end;
}
ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts,
AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output pixel format\n");
goto end;
}
/*
* Set the endpoints for the filter graph. The filter_graph will
* be linked to the graph described by filters_descr.
*/
/*
* The buffer source output must be connected to the input pad of
* the first filter described by filters_descr; since the first
* filter input label is not specified, it is set to "in" by
* default.
*/
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
/*
* The buffer sink input must be connected to the output pad of
* the last filter described by filters_descr; since the last
* filter output label is not specified, it is set to "out" by
* default.
*/
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr,
&inputs, &outputs, NULL)) < 0)
goto end;
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
goto end;
end:
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return ret;
}
static void display_frame(const AVFrame *frame, AVRational time_base)
{
int x, y;
uint8_t *p0, *p;
int64_t delay;
if (frame->pts != AV_NOPTS_VALUE) {
if (last_pts != AV_NOPTS_VALUE) {
/* sleep roughly the right amount of time;
* usleep is in microseconds, just like AV_TIME_BASE. */
delay = av_rescale_q(frame->pts - last_pts,
time_base, AV_TIME_BASE_Q);
if (delay > 0 && delay < 1000000)
usleep(delay);
}
last_pts = frame->pts;
}
/* Trivial ASCII grayscale display. */
p0 = frame->data[0];
puts("\033c");
for (y = 0; y < frame->height; y++) {
p = p0;
for (x = 0; x < frame->width; x++)
putchar(" .-+#"[*(p++) / 52]);
putchar('\n');
p0 += frame->linesize[0];
}
fflush(stdout);
}
int main(int argc, char **argv)
{
int ret;
AVPacket packet;
AVFrame *frame;
AVFrame *filt_frame;
if (argc != 2) {
fprintf(stderr, "Usage: %s file\n", argv[0]);
exit(1);
}
frame = av_frame_alloc();
filt_frame = av_frame_alloc();
if (!frame || !filt_frame) {
perror("Could not allocate frame");
exit(1);
}
if ((ret = open_input_file(argv[1])) < 0)
goto end;
if ((ret = init_filters(filter_descr)) < 0)
goto end;
/* read all packets */
while (1) {
if ((ret = av_read_frame(fmt_ctx, &packet)) < 0)
break;
if (packet.stream_index == video_stream_index) {
ret = avcodec_send_packet(dec_ctx, &packet);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while sending a packet to the decoder\n");
break;
}
while (ret >= 0) {
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
} else if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while receiving a frame from the decoder\n");
goto end;
}
frame->pts = frame->best_effort_timestamp;
/* push the decoded frame into the filtergraph */
if (av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while feeding the filtergraph\n");
break;
}
/* pull filtered frames from the filtergraph */
while (1) {
ret = av_buffersink_get_frame(buffersink_ctx, filt_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
if (ret < 0)
goto end;
display_frame(filt_frame, buffersink_ctx->inputs[0]->time_base);
av_frame_unref(filt_frame);
}
av_frame_unref(frame);
}
}
av_packet_unref(&packet);
}
end:
avfilter_graph_free(&filter_graph);
avcodec_free_context(&dec_ctx);
avformat_close_input(&fmt_ctx);
av_frame_free(&frame);
av_frame_free(&filt_frame);
if (ret < 0 && ret != AVERROR_EOF) {
fprintf(stderr, "Error occurred: %s\n", av_err2str(ret));
exit(1);
}
exit(0);
}
That sample uses these filters ;
"scale=78:24,transpose=cclock"
I compiled and run it with a sample video file but it just outputs fancy characters to console, the code block given below is responsible for this ;
/* Trivial ASCII grayscale display. */
p0 = frame->data[0];
puts("\033c");
for (y = 0; y < frame->height; y++) {
p = p0;
for (x = 0; x < frame->width; x++)
putchar(" .-+#"[*(p++) / 52]);
putchar('\n');
p0 += frame->linesize[0];
}
fflush(stdout);
I have no issues with Encoding & Decoding, i just don't know how to apply "overlay" filter. Are there any tutorials out there demonstrate how to use "overlay" filter?
Just like in the example, except you use "overlay".
snprintf(args, sizeof(args), args here...);
avfilter_graph_create_filter(sink, avfilter_get_by_name("overlay"), nullptr, nullptr, arg, graph);
Then you need TWO create two source pads. i.e.
avfilter_graph_create_filter(sourceX, avfilter_get_by_name("buffer"), nullptr, args, nullptr, m_graph);
and one sink pad. Then feed one source with the video frame and other second with the image to overlay
Following code fragments will be helpful..
char args[512];
int ret = 0;
const AVFilter *bufferSrc = avfilter_get_by_name("buffer");
const AVFilter *bufferOvr = avfilter_get_by_name("buffer");
const AVFilter *bufferSink = avfilter_get_by_name("buffersink");
const AVFilter *ovrFilter = avfilter_get_by_name("overlay");
const AVFilter *colorFilter = avfilter_get_by_name("colorchannelmixer");
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE };
fFilterGraph = avfilter_graph_alloc();
if (!fFilterGraph) {
ret = AVERROR(ENOMEM);
goto end;
}
/* buffer video source: the decoded frames from the decoder will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
decCtx->width, decCtx->height, decCtx->pix_fmt,
fTimeBase.num, fTimeBase.den,
decCtx->sample_aspect_ratio.num, decCtx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&fBufSrc0Ctx, bufferSrc, "in0",
args, NULL, fFilterGraph);
if (ret < 0)
goto end;
/* buffer video overlay source: the overlayed frame from the file will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
ovrCtx->width, ovrCtx->height, ovrCtx->pix_fmt,
fTimeBase.num, fTimeBase.den,
ovrCtx->sample_aspect_ratio.num, ovrCtx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&fBufSrc1Ctx, bufferOvr, "in1",
args, NULL, fFilterGraph);
if (ret < 0)
goto end;
/* color filter */
snprintf(args, sizeof(args), "aa=%f", (float)fWatermarkOpacity / 10.0);
ret = avfilter_graph_create_filter(&fColorFilterCtx, colorFilter, "colorFilter",
args, NULL, fFilterGraph);
if (ret < 0)
goto end;
/* overlay filter */
switch (fWatermarkPos) {
case 0:
/* Top left */
snprintf(args, sizeof(args), "x=%d:y=%d:repeatlast=1",
fWatermarkOffset, fWatermarkOffset);
break;
case 1:
/* Top right */
snprintf(args, sizeof(args), "x=W-w-%d:y=%d:repeatlast=1",
fWatermarkOffset, fWatermarkOffset);
break;
case 3:
/* Bottom left */
snprintf(args, sizeof(args), "x=%d:y=H-h-%d:repeatlast=1",
fWatermarkOffset, fWatermarkOffset);
break;
case 4:
/* Bottom right */
snprintf(args, sizeof(args), "x=W-w-%d:y=H-h-%d:repeatlast=1",
fWatermarkOffset, fWatermarkOffset);
break;
case 2:
default:
/* Center */
snprintf(args, sizeof(args), "x=(W-w)/2:y=(H-h)/2:repeatlast=1");
break;
}
ret = avfilter_graph_create_filter(&fOvrFilterCtx, ovrFilter, "overlay",
args, NULL, fFilterGraph);
if (ret < 0)
goto end;
/* buffer sink - destination of the final video */
ret = avfilter_graph_create_filter(&fBufSinkCtx, bufferSink, "out",
NULL, NULL, fFilterGraph);
if (ret < 0)
goto end;
ret = av_opt_set_int_list(fBufSinkCtx, "pix_fmts", pix_fmts,
AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0)
goto end;
/*
* Link all filters..
*/
avfilter_link(fBufSrc0Ctx, 0, fOvrFilterCtx, 0);
avfilter_link(fBufSrc1Ctx, 0, fColorFilterCtx, 0);
avfilter_link(fColorFilterCtx, 0, fOvrFilterCtx, 1);
avfilter_link(fOvrFilterCtx, 0, fBufSinkCtx, 0);
if ((ret = avfilter_graph_config(fFilterGraph, NULL)) < 0)
goto end;
end:
i try to encode (with compression) and decode (without compression) a image with ffmpeg. But if i want to get the sent image back with avcodec_receive_packet i get only the error AVERROR(EAGAIN).
It doesnt matter what i change ... allways AVERROR(EAGAIN) is the outcome. Is it maybe a problem of sending just one single frame to the encoder? And if yes, how to fix it?
Code (only relevant stuff shown):
avcodec_register_all();
/* ------ init codec ------------------*/
AVCodec *codec;
codec = avcodec_find_decoder(AV_CODEC_ID_H264);
if (!codec)
{
print("compressH264, could not find decoder:\"AV_CODEC_ID_H264\"!!!");
return false;
}
AVCodec *nVidiaCodec = avcodec_find_encoder_by_name("h264_nvenc");
if (!nVidiaCodec)
{
print("err");
}
/* ------ ------------ ------------------*/
/* ------ init context ------------------*/
AVCodecContext* av_codec_context_ = NULL;
av_codec_context_ = avcodec_alloc_context3(nVidiaCodec);
if (!av_codec_context_)
{
print("compressH264, avcodec_alloc_context3 failed!!!");
return false;
}
int w = imgSrc.width();
int h = imgSrc.height();
if ((w % 2) != 0)
{
++w;
}
if ((h % 2) != 0)
{
++h;
}
av_codec_context_->width = w;
av_codec_context_->height = h;
av_codec_context_->pix_fmt = AV_PIX_FMT_YUV420P;
av_codec_context_->gop_size = 1;
av_codec_context_->max_b_frames = 1;
av_codec_context_->bit_rate = 400000;
av_codec_context_->time_base.den = 1;
av_codec_context_->time_base.num = 1;
av_opt_set(av_codec_context_->priv_data, "preset", "slow", 0);
int ret = avcodec_open2(av_codec_context_, nVidiaCodec, NULL);
if (0 > ret)
{
print("compressH264, could not open codec context for decoder:\"AV_CODEC_ID_H264\"!!!");
return false;
}
AVFrame *picture = av_frame_alloc();
picture->format = AV_PIX_FMT_RGB24;
picture->width = w;
picture->height = h;
ret = avpicture_fill((AVPicture *)picture, imgSrc.bits(), AV_PIX_FMT_RGB24, w, h);
if (0 > ret)
{
print("compressH264, avpicture_fill - failed!!!");
return false;
}
AVFrame *tmp_picture = av_frame_alloc();
tmp_picture->format = AV_PIX_FMT_YUV420P;
tmp_picture->width = w;
tmp_picture->height = h;
ret = av_frame_get_buffer(tmp_picture, 32);
SwsContext *img_convert_ctx = sws_getContext(av_codec_context_->width, av_codec_context_->height, AV_PIX_FMT_RGB24, av_codec_context_->width, av_codec_context_->height, av_codec_context_->pix_fmt, SWS_BICUBIC, NULL, NULL, NULL);
ret = sws_scale(img_convert_ctx, picture->data, picture->linesize, 0, av_codec_context_->height, tmp_picture->data, tmp_picture->linesize);
int h264Size = avpicture_get_size(AV_PIX_FMT_YUV420P, w, h);
ret = avcodec_send_frame(av_codec_context_, tmp_picture);
if (0 > ret)
{
char err[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(err, AV_ERROR_MAX_STRING_SIZE, ret);
print("compressH264, avcodec_send_frame: %s", err);
}
AVPacket *pkt = av_packet_alloc();
while (ret >= 0)
{
ret = avcodec_receive_packet(av_codec_context_, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
{
break;
}
else if (ret < 0)
{
fprintf(stderr, "Error during encoding\n");
exit(1);
}
av_packet_unref(pkt);
}
print("success");
Everything works well until:
- avcodec_receive_packet ... i get all time the error AVERROR(EAGAIN).
I can start decoding just if i have the compressed image.
Thanks for your help guys.
Edit:
If i do now the following code, i get a packet and ret == 0, but i have to send 46 times the same image ... for me this makes no sence.
do
{
ret = avcodec_receive_packet(av_codec_context_, &pkt);
if (ret == 0)
{
break;
}
else if ((ret < 0) && (ret != AVERROR(EAGAIN)))
{
coutF("error");
}
else if (ret == AVERROR(EAGAIN))
{
ret = avcodec_send_frame(av_codec_context_, tmp_picture);
if (0 > ret)
{
char err[AV_ERROR_MAX_STRING_SIZE];
av_make_error_string(err, AV_ERROR_MAX_STRING_SIZE, ret);
coutFRed("compressH264, avcodec_send_frame: %s", err);
}
coutF("cnt:%d", ++cnt);
}
} while (ret == 0);
Edit:
Good morning,
after more invest, i got the issue. I have to send the same frame a lot of time, because of the keyframe stuff for h264. The question now is, if it is possible to remove the h264 standart stuff from the encoder and just let FFMPEG convert one single frame.
I am not sure but following an ffmpeg example it seems that that just means it is done and you should return like they do in this code snippet:
/* if no more frames for output - returns AVERROR(EAGAIN)
* if flushed and no more frames for output - returns AVERROR_EOF
* rewrite retcode to 0 to show it as normal procedure completion
*/
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
ret = 0;
In their comments they seem to imply that it signals "normal procedure completion"
the answer for this issue is this:
do
{
ret = avcodec_receive_packet(av_codec_context_, &pkt);
if (ret == 0)
{
break;
}
else if ((ret < 0) && (ret != AVERROR(EAGAIN)))
{
return false;
}
else if (ret == AVERROR(EAGAIN))
{
ret = avcodec_send_frame(av_codec_context_, NULL);
if (0 > ret)
{
return false;
}
}
} while (ret == 0);
The NULL frame will flush the buffer and we are able to get the encoded frame.
I want to create an animated gif in a QT project.
When I scale the a QImage to AV_PIX_FMT_RGB8 directly the output looks awfull with flickering artifacts and when I scale to AV_PIX_FMT_YUV420P in between the output is dithered which does not look much better.
I found out that ffmpeg is able to produce a palette file using a filter called palettegen and then convert a movie to gif using this palette.
Is there any sample c++ file out which I could use for my project or does anybody have a clue who to use these filters in code?
The filter can be describe like below
format=pix_fmts=rgb32,fps=10,scale=320:240:flags=lanczos,split [o1] [o2];[o1] palettegen [p]; [o2] fifo [o3];[o3] [p] paletteuse
You can do some fps or scale customization with the filter above.
Then we should apply this filter graph in ffmpeg.
One important note is, as the filter palettegen need the entire stream, we need to add the whole stream to the buffer source, and a end of stream to buffer source, then we can get frame from buffer sink and muxing to the output gif file.
Here is a completed example, change the micro definition in your test and be sure to use the latest version of ffmpeg.
#define INPUT_PATH "/Users/gif/Downloads/VUE_1503566813005.mp4"
#define OUTPUT_PATH "/Users/gif/Downloads/VUE.gif"
video2gif.cpp
#include <iostream>
#ifdef __cplusplus
extern "C" {
#include "libavformat/avformat.h"
#include "libavfilter/avfilter.h"
#include "libavutil/opt.h"
#include "libavfilter/buffersrc.h"
#include "libavfilter/buffersink.h"
#include "libswscale/swscale.h"
#include "libavutil/imgutils.h"
}
#endif
#define INPUT_PATH "/Users/gif/Downloads/VUE_1503566813005.mp4"
#define OUTPUT_PATH "/Users/gif/Downloads/VUE.gif"
const char *filter_descr = "format=pix_fmts=rgb32,fps=10,scale=320:240:flags=lanczos,split [o1] [o2];[o1] palettegen [p]; [o2] fifo [o3];[o3] [p] paletteuse";
static AVFormatContext* ofmt_ctx;
static AVCodecContext* o_codec_ctx;
static AVFilterGraph* filter_graph;
static AVFilterContext* buffersrc_ctx;
static AVFilterContext* buffersink_ctx;
static int init_filters(const char* filter_desc,
AVFormatContext* ifmt_ctx,
int stream_index,
AVCodecContext* dec_ctx)
{
char args[512];
int ret = 0;
AVFilter* buffersrc = avfilter_get_by_name("buffer");
AVFilter* buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut* inputs = avfilter_inout_alloc();
AVFilterInOut* outputs = avfilter_inout_alloc();
AVRational time_base = ifmt_ctx->streams[stream_index]->time_base;
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_PAL8, AV_PIX_FMT_NONE };
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
return ret;
}
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in", args, nullptr, filter_graph);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "Cannot create buffer source\n");
return ret;
}
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out", nullptr, nullptr, filter_graph);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "Cannot create buffer sink\n");
return ret;
}
av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "can not set output pixel format\n");
return ret;
}
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = nullptr;
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = nullptr;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filter_desc, &inputs, &outputs, nullptr)) < 0) {
av_log(nullptr, AV_LOG_ERROR, "parse filter graph error\n");
return ret;
}
if ((ret = avfilter_graph_config(filter_graph, nullptr)) < 0) {
av_log(nullptr, AV_LOG_ERROR, "config graph error\n");
}
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return 0;
}
static int init_muxer()
{
int ret = 0;
AVOutputFormat* o_fmt = av_guess_format("gif", OUTPUT_PATH, "video/gif");
ret = avformat_alloc_output_context2(&ofmt_ctx, o_fmt, "gif", OUTPUT_PATH);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "%s allocate output format\n", av_err2str(ret));
return -1;
}
AVCodec* codec = avcodec_find_encoder(AV_CODEC_ID_GIF);
if (!codec) {
return -1;
}
AVStream* stream = avformat_new_stream(ofmt_ctx, codec);
AVCodecParameters* codec_paramters = stream->codecpar;
codec_paramters->codec_tag = 0;
codec_paramters->codec_id = codec->id;
codec_paramters->codec_type = AVMEDIA_TYPE_VIDEO;
codec_paramters->width = 320;
codec_paramters->height = 240;
codec_paramters->format = AV_PIX_FMT_PAL8;
o_codec_ctx = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(o_codec_ctx, codec_paramters);
o_codec_ctx->time_base = { 1, 10 };
if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER) {
o_codec_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
ret = avcodec_open2(o_codec_ctx, codec, NULL);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "%s open output codec\n", av_err2str(ret));
return ret;
}
ret = avio_open(&ofmt_ctx->pb, OUTPUT_PATH, AVIO_FLAG_WRITE);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "%s avio open error\n", av_err2str(ret));
return ret;
}
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "%s write header\n", av_err2str(ret));
return ret;
}
av_dump_format(ofmt_ctx, -1, OUTPUT_PATH, 1);
return 0;
}
static void destroy_muxer()
{
avformat_free_context(ofmt_ctx);
avcodec_close(o_codec_ctx);
avcodec_free_context(&o_codec_ctx);
}
static void destroy_filter()
{
avfilter_free(buffersrc_ctx);
avfilter_free(buffersink_ctx);
avfilter_graph_free(&filter_graph);
}
static void muxing_one_frame(AVFrame* frame)
{
int ret = avcodec_send_frame(o_codec_ctx, frame);
AVPacket *pkt = av_packet_alloc();
av_init_packet(pkt);
while (ret >= 0) {
ret = avcodec_receive_packet(o_codec_ctx, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
av_write_frame(ofmt_ctx, pkt);
}
av_packet_unref(pkt);
}
int main(int argc, const char * argv[]) {
av_register_all();
avcodec_register_all();
avfilter_register_all();
int ret = 0;
AVFormatContext* fmt_ctx = avformat_alloc_context();
AVCodecContext* codec_ctx = nullptr;
AVCodec* codec = nullptr;
int video_index = -1;
AVPacket *pkt = av_packet_alloc();
av_init_packet(pkt);
pkt->size = 0;
pkt->data = nullptr;
ret = avformat_open_input(&fmt_ctx, INPUT_PATH, NULL, NULL);
if (ret != 0) {
std::cerr << "error open input" << std::endl;
goto die;
}
avformat_find_stream_info(fmt_ctx, NULL);
av_dump_format(fmt_ctx, -1, INPUT_PATH, 0);
ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &codec, 0);
if (ret == AVERROR_STREAM_NOT_FOUND) {
std::cerr << "error no video stream found" << std::endl;
goto die;
}
for (unsigned i = 0; i < fmt_ctx->nb_streams; i++) {
if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
video_index = i;
if (ret == AVERROR_DECODER_NOT_FOUND) {
codec = avcodec_find_decoder(fmt_ctx->streams[i]->codecpar->codec_id);
}
break;
}
}
if (!codec) {
std::cerr << "could not find the decoder" << std::endl;
goto die;
}
codec_ctx = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(codec_ctx, fmt_ctx->streams[video_index]->codecpar);
ret = avcodec_open2(codec_ctx, codec, NULL);
if (ret < 0) {
std::cerr << "open codec error" << std::endl;
goto die;
}
if (init_muxer() < 0) {
av_log(nullptr, AV_LOG_ERROR, "could not init muxer\n");
goto die;
}
if ((ret = init_filters(filter_descr, fmt_ctx, video_index, codec_ctx)) < 0) {
av_log(nullptr, AV_LOG_ERROR, "could not init filters %s\n", av_err2str(ret));
goto die;
}
// it's time to decode
while (av_read_frame(fmt_ctx, pkt) == 0) {
if (pkt->stream_index == video_index) {
ret = avcodec_send_packet(codec_ctx, pkt);
while (ret >= 0) {
AVFrame* frame = av_frame_alloc();
ret = avcodec_receive_frame(codec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
if (ret >= 0) {
// processing one frame
frame->pts = frame->best_effort_timestamp;
// palettegen need a whole stream, just add frame to buffer and don't get frame
ret = av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "error add frame to buffer source %s\n", av_err2str(ret));
}
}
av_frame_free(&frame);
}
}
}
// end of buffer
if ((ret = av_buffersrc_add_frame_flags(buffersrc_ctx, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF)) >= 0) {
do {
AVFrame* filter_frame = av_frame_alloc();
ret = av_buffersink_get_frame(buffersink_ctx, filter_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
// av_log(nullptr, AV_LOG_ERROR, "error get frame from buffer sink %s\n", av_err2str(ret));
av_frame_unref(filter_frame);
break;
}
// write the filter frame to output file
muxing_one_frame(filter_frame);
av_log(nullptr, AV_LOG_INFO, "muxing one frame\n");
av_frame_unref(filter_frame);
} while (ret >= 0);
} else {
av_log(nullptr, AV_LOG_ERROR, "error add frame to buffer source %s\n", av_err2str(ret));
}
av_packet_free(&pkt);
av_write_trailer(ofmt_ctx);
destroy_muxer();
destroy_filter();
die:
avformat_free_context(fmt_ctx);
avcodec_close(codec_ctx);
avcodec_free_context(&codec_ctx);
return 0;
}
I'm trying to encode a set of pictures into a video with ffmpeg. I'm new to this stuff, but I managed to get it work. I have only one problem: the first second or two of video look good but as the time goes the video quality keeps dropping and dropping. At the end (it's a video of about 16 seconds) the quality is really bad and I can't understand why.
I'm using AV_CODEC_ID_MPEG1VIDEO as video codec (frankly, it's the only one I could make work) and here is a sample of my code:
c = avcodec_alloc_context3(codec);
if (!c) {
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
c->bit_rate = 400000;
c->width = width;
c->height = height;
struct AVRational time_base = {1, 25};
c->time_base = time_base;
c->gop_size = 10;
c->max_b_frames = 1;
c->pix_fmt = AV_PIX_FMT_YUV420P;
if (codec_id == AV_CODEC_ID_H264)
av_opt_set(c->priv_data, "preset", "slow", 0);
// Open the codec
if (avcodec_open2(c, codec, NULL) < 0) {
fprintf(stderr, "Could not open codec\n");
exit(1);
}
fopen_s(&f, filename, "wb");
if (!f) {
fprintf(stderr, "Could not open %s\n", filename);
exit(1);
}
frame = av_frame_alloc();
if (!frame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
frame->format = c->pix_fmt;
frame->width = c->width;
frame->height = c->height;
ret = av_image_alloc(frame->data, frame->linesize, c->width, c->height,
c->pix_fmt, 32);
if (ret < 0) {
fprintf(stderr, "Could not allocate raw picture buffer\n");
exit(1);
}
struct SwsContext *frameConvertContext = sws_getContext(width, height,
PIX_FMT_RGB24,
c->width, c->height,
c->pix_fmt,
SWS_LANCZOS | SWS_ACCURATE_RND, NULL, NULL, NULL);
for (i = 0; i < framesNr; i++) {
// Read an image
std::stringstream filename;
filename << "input/image-" << (i+1) << ".jpg";
Image* img;
img = Image::fromJpeg((char*) filename.str().c_str());
int srcSliceY[1] = { img->getWidth() * 3 };
const uint8_t* imgbuf[1] = { (uint8_t*) img->getData(sizeof(uint8_t)) };
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
fflush(stdout);
frameConvertContext = sws_getCachedContext(frameConvertContext, width, height,
PIX_FMT_RGB24,
c->width, c->height,
c->pix_fmt,
SWS_LANCZOS | SWS_ACCURATE_RND, NULL, NULL, NULL);
sws_scale(frameConvertContext, imgbuf, srcSliceY, 0, img->getHeight(), frame->data, frame->linesize);
frame->pts = i;
/* encode the image */
ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
if (ret < 0) {
fprintf(stderr, "Error encoding frame\n");
exit(1);
}
if (got_output) {
printf(".");
fwrite(pkt.data, 1, pkt.size, f);
av_free_packet(&pkt);
}
// Free the memory
delete img;
}
Any tips? Thanks a lot!
400kb mpeg 1? Yes the quality will be abysmal. Use a higher bitrate or a better codec.