Get video from webcam using FFmpeg Libav - c++

I am trying to record webcam video using FFmpeg C libraries (libav), on a Mac computer. I made changes to the transcode.c example so that it opens a device instead of a file. However, for some reason the code only receives a single packet and then closes.
static int open_input_source(const char *dev_name) {
int ret;
unsigned int i;
AVDictionary *p_av_options = NULL;
AVInputFormat *p_av_input_format = av_find_input_format("avfoundation");
av_dict_set(&p_av_options, "framerate", "30", 0);
ifmt_ctx = NULL;
if ((ret = avformat_open_input(&ifmt_ctx, dev_name, p_av_input_format, &p_av_options) < 0)) {
av_log(NULL, AV_LOG_ERROR, "Cannot open input file\n");
return ret;
}
if ((ret = avformat_find_stream_info(ifmt_ctx, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n");
return ret;
}
stream_ctx = av_calloc(ifmt_ctx->nb_streams, sizeof(*stream_ctx));
if (!stream_ctx)
return AVERROR(ENOMEM);
for (i = 0; i < ifmt_ctx->nb_streams; i++) {
AVStream *stream = ifmt_ctx->streams[i];
const AVCodec *dec = avcodec_find_decoder(stream->codecpar->codec_id);
AVCodecContext *codec_ctx;
if (!dec) {
av_log(NULL, AV_LOG_ERROR, "Failed to find decoder for stream #%u\n", i);
return AVERROR_DECODER_NOT_FOUND;
}
codec_ctx = avcodec_alloc_context3(dec);
if (!codec_ctx) {
av_log(NULL, AV_LOG_ERROR, "Failed to allocate the decoder context for stream #%u\n", i);
return AVERROR(ENOMEM);
}
ret = avcodec_parameters_to_context(codec_ctx, stream->codecpar);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Failed to copy decoder parameters to input decoder context "
"for stream #%u\n", i);
return ret;
}
/* Reencode video & audio and remux subtitles etc. */
if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO
|| codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) {
if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO)
codec_ctx->framerate = av_guess_frame_rate(ifmt_ctx, stream, NULL);
/* Open decoder */
ret = avcodec_open2(codec_ctx, dec, NULL);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Failed to open decoder for stream #%u\n", i);
return ret;
}
}
stream_ctx[i].dec_ctx = codec_ctx;
stream_ctx[i].dec_frame = av_frame_alloc();
if (!stream_ctx[i].dec_frame)
return AVERROR(ENOMEM);
}
av_dump_format(ifmt_ctx, 0, dev_name, 0);
return 0;
}
I have looked for other code examples but they are all deprecated and no longer compile in updated FFmpeg.
Is there some missing setting in my open_input_source function? Alternatively, is the problem in using transcoding is my basis? Should I try to use some other example?
In general, is there is a C source code reference which fulfills my requirements?
Thanks

This is a pretty fleshed out example that might meet your requirements:
https://github.com/argvk/ffmpeg-examples/blob/master/dshow_capture_video.c
I don't think you need nearly as much code as is include there, but you might be able to just update lines 260 with how long you want it to run (that example is 300 frames) and line 83 to open your webcam (sounds like you've already successfully done this with ret = avformat_open_input(&ifmt_ctx, dev_name, p_av_input_format, &p_av_options) in your code.
There are lots of other options there which you might want to remove, keep, or change depending on the details which are not provided here. Unfortunately I don't have a simplified code sample I'm able to share on here, but this dshow example is doing everything I expect.
Hope it helps some.

Related

ffmpeg memory leak in the avcodec_open2 method

I've developed an application which handles live video stream. The problem is that it should run as a service and over time I am noticing some memory increase. When I check the application with valgrind - it did not find any leak related issues.
So I've check it with google profile tools. This is a result(substracting the one of the first dumps from the latest) after approximately 6 hour run:
30.0 35.7% 35.7% 30.0 35.7% av_malloc
28.9 34.4% 70.2% 28.9 34.4% av_reallocp
24.5 29.2% 99.4% 24.5 29.2% x264_malloc
When I check the memory on the graph I see, that these allocations are related to avcodec_open2. The client code is:
` g_EncoderMutex.lock();
ffmpeg_encoder_start(OutFileName.c_str(), AV_CODEC_ID_H264, m_FPS, width, height);
for (pts = 0; pts < VideoImages.size(); pts++) {
m_frame->pts = pts;
ffmpeg_encoder_encode_frame(VideoImages[pts].RGBimage[0]);
}
ffmpeg_encoder_finish();
g_EncoderMutex.unlock()
The ffmpeg_encoder_start method is:
void VideoEncoder::ffmpeg_encoder_start(const char *filename, int codec_id, int fps, int width, int height)
{
int ret;
m_FPS=fps;
AVOutputFormat * fmt = av_guess_format(filename, NULL, NULL);
m_oc = NULL;
avformat_alloc_output_context2(&m_oc, NULL, NULL, filename);
m_stream = avformat_new_stream(m_oc, 0);
AVCodec *codec=NULL;
codec = avcodec_find_encoder(codec_id);
if (!codec)
{
fprintf(stderr, "Codec not found\n");
return; //-1
}
m_c=m_stream->codec;
avcodec_get_context_defaults3(m_c, codec);
m_c->bit_rate = 400000;
m_c->width = width;
m_c->height = height;
m_c->time_base.num = 1;
m_c->time_base.den = m_FPS;
m_c->gop_size = 10;
m_c->max_b_frames = 1;
m_c->pix_fmt = AV_PIX_FMT_YUV420P;
if (codec_id == AV_CODEC_ID_H264)
av_opt_set(m_c->priv_data, "preset", "ultrafast", 0);
if (m_oc->oformat->flags & AVFMT_GLOBALHEADER)
m_c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
avcodec_open2( m_c, codec, NULL );
m_stream->time_base=(AVRational){1, m_FPS};
if (avio_open(&m_oc->pb, filename, AVIO_FLAG_WRITE) < 0)
{
printf( "Could not open '%s'\n", filename);
exit(1);
}
avformat_write_header(m_oc, NULL);
m_frame = av_frame_alloc();
if (!m_frame) {
printf( "Could not allocate video frame\n");
exit(1);
}
m_frame->format = m_c->pix_fmt;
m_frame->width = m_c->width;
m_frame->height = m_c->height;
ret = av_image_alloc(m_frame->data, m_frame->linesize, m_c->width, m_c->height, m_c->pix_fmt, 32);
if (ret < 0) {
printf("Could not allocate raw picture buffer\n");
exit(1);
}
}
The ffmpeg_encoder_encode_frame is:
void VideoEncoder::ffmpeg_encoder_encode_frame(uint8_t *rgb)
{
int ret, got_output;
ffmpeg_encoder_set_frame_yuv_from_rgb(rgb);
av_init_packet(&m_pkt);
m_pkt.data = NULL;
m_pkt.size = 0;
ret = avcodec_encode_video2(m_c, &m_pkt, m_frame, &got_output);
if (ret < 0) {
printf("Error encoding frame\n");
exit(1);
}
if (got_output)
{
av_packet_rescale_ts(&m_pkt,
(AVRational){1, m_FPS}, m_stream->time_base);
m_pkt.stream_index = m_stream->index;
int ret = av_interleaved_write_frame(m_oc, &m_pkt);
av_packet_unref(&m_pkt);
}
}
ffmpeg_encoder_finish code is:
void VideoEncoder::ffmpeg_encoder_finish(void)
{
int got_output, ret;
do {
ret = avcodec_encode_video2(m_c, &m_pkt, NULL, &got_output);
if (ret < 0) {
printf( "Error encoding frame\n");
exit(1);
}
if (got_output) {
av_packet_rescale_ts(&m_pkt,
(AVRational){1, m_FPS}, m_stream->time_base);
m_pkt.stream_index = m_stream->index;
int ret = av_interleaved_write_frame(m_oc, &m_pkt);
av_packet_unref(&m_pkt);
}
} while (got_output);
av_write_trailer(m_oc);
avio_closep(&m_oc->pb);
avformat_free_context(m_oc);
av_freep(&m_frame->data[0]);
av_frame_free(&m_frame);
av_packet_unref(&m_pkt);
sws_freeContext(m_sws_context);
}
This code runs multiple times in the loop.
So my question is - what am I doing wrong? maybe ffmpeg is using some kind of internal buffering? If so, how to disable it? Because such an increase in memory usage is unacceptable at all.
You didn't close encoder context. Add avcodec_close(m_c) to ffmpeg_encoder_finish().
See ffmpeg.org
User is required to call avcodec_close() and avformat_free_context() to clean up the allocation by avformat_new_stream().
Plus I don't see how m_c is allocated. Usually it is allocated with avcodec_alloc_context and must be deallocated with av_free (after closing of course).
Don't use valgrind to check memory leaks for your own projects, use sanitizers, with these you can pin point the source of the leak. Check this out: Multi-Threaded Video Decoder Leaks Memory
Hope that helps.
It's sufficient to call 'avcodec_free_context(m_c)', this procedure calls 'avcodec_close' and also de-allocates 'extradata'(if it's was allocated) and 'subtitle_header' (if it was allocated).

c++ videoplayer ffmpeg: get pixel data?

I want to get the pixel data of a frame. I found this (in original version as old code) and changed some things.
I have this code:
AVFormatContext *pFormatCtx;
pFormatCtx = avformat_alloc_context();
// Open file
if (int err = avformat_open_input(&pFormatCtx, file, NULL, 0) != 0)
{
exit(2);
}
// Get infromation about streams
if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
{
exit(2);
}
// # video stream
int videoStreamIndex = -1;
AVCodecContext *pVideoCodecCtx;
AVCodec *pVideoCodec;
int res = 0;
int width = 0;
int height = 0;
for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++)
{
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
{
videoStreamIndex = i;
pVideoCodecCtx = pFormatCtx->streams[i]->codec;
// Find decoder
pVideoCodec = avcodec_find_decoder(pVideoCodecCtx->codec_id);
if (pVideoCodec)
{
// Open decoder
res = !(avcodec_open2(pVideoCodecCtx, pVideoCodec, NULL) < 0);
width = pVideoCodecCtx->coded_width;
height = pVideoCodecCtx->coded_height;
}
break;
}
}
// Frame width
width = pFormatCtx->streams[videoStreamIndex]->codec->width;
// Frame height
height = pFormatCtx->streams[videoStreamIndex]->codec->height;
AVPacket packet;
int got_picture_ptr;
AVPacket *avpkt;
AVFrame * pOutFrame;
pOutFrame = av_frame_alloc();
AVFrame * rgbOutFrame = av_frame_alloc();
if (!pOutFrame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
while (av_read_frame(pFormatCtx, &packet) >= 0)
{
if (packet.stream_index == videoStreamIndex)
{
// Decode packeg to frame.
int videoFrameBytes = avcodec_decode_video2(pVideoCodecCtx, pOutFrame,
&got_picture_ptr, &packet);
// Create context
SwsContext* pImgConvertCtx = sws_getContext(pVideoCodecCtx->width,
pVideoCodecCtx->height,
pVideoCodecCtx->pix_fmt,
pVideoCodecCtx->width, pVideoCodecCtx->height,
AV_PIX_FMT_YUV420P,
SWS_BICUBIC, NULL, NULL, NULL);
// Convert frame
sws_scale(pImgConvertCtx, pOutFrame->data, pOutFrame->linesize,
width, height, rgbOutFrame->data, rgbOutFrame->linesize);
}
}
I know, that the code from SwsContext and sws_scale is wrong but I wonder, where can I find the pixel data of my frame... (and in which format it is stored).
Can someone help me here?
Pixel data is stored in data field.
According to the documentation:
uint8_t* AVFrame::data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Look here for more information.
Generally speaking, your code is a bit misleading and rather buggy. I can point out some drawbacks:
1) You don't need to create new SwsContext on every incoming video packet. Just create it once before while cycle.
2) Next, you have an rgbOutFrame, but SwsContext is created for scaling into the YUV420 pixel format. It looks strange.
3) Besides, avcodec_decode_video2 is invoked, but you never check neither return value nor got_picture_ptr flag. Such practice is really error-prone.
And so on...
Hope it'll help you to improve your program and get necessary results.

RTP server for audio stream

I'm in the mid of my project where I'm trying to send the audio data over RTP using ffmpeg in VC++.
but I'm unable to do that.
My start up code is:
setRTPOutputAddr("127.0.0.1");
setRTPOutputPort(9985);
av_register_all();
avformat_network_init();
formatCtx= avformat_alloc_context();
//av_dump_format(formatCtx,0,"rtp",1);
avio_open_dyn_buf(&ioctx);
avio_open(&formatCtx->pb, "rtp", AVIO_FLAG_WRITE);
//av_dump_format(formatCtx,0,"rtp",1);
formatCtx->pb = ioctx;
rtpCtx = avformat_alloc_context();
rtpCtx->oformat = av_guess_format("rtp",0,0);
AVStream *fakeVideo = 0;
fakeVideo = avformat_new_stream(rtpCtx,0);
avcodec_get_context_defaults3(fakeVideo->codec, NULL);
fakeVideo->codec->codec_id = AV_CODEC_ID_MPEG2TS;
rtpCtx->audio_codec_id = AV_CODEC_ID_NONE;
rtpCtx->video_codec_id = AV_CODEC_ID_MPEG2TS;
// avformat_write_header(rtpCtx,0);
char *url = new char[1024];
sprintf(url,"rtp://%s:%d",rtpOutputAddr,rtpOutputPort);
printf("will output to url:%s\n",url);
avio_open(&rtpCtx->pb,url,AVIO_FLAG_WRITE);
avformat_write_header(rtpCtx, NULL);
delete url;
if (avio_open(&formatCtx->pb, "rtp", AVIO_FLAG_WRITE) < 0)
{
fprintf(stderr, "Could not open '%s'\n", formatCtx->filename);
return -1;
}
else
{
printf("succeed \n");
}
Error Here: avformat_write_header(formatCtx, NULL); //program Crashes
Do anyone having any idea where I'm doing wrong?
Thanks!

Problems converting .flv to mp3 using FFmpeg SDK

I'm using the FFmpeg SDK to programmatically convert videos to mp3.
I read the audio frames of the video this way:
while(av_read_frame(pFromCtx, &pkt) >= 0)
{
if(pkt.stream_index == audioStreamIndex)
{
avcodec_get_frame_defaults(frame);
got_frame = 0;
ret = avcodec_decode_audio4(pCodecCtx, frame, &got_frame, &pkt);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error decoding audio frame.\n");
continue;
}
if(got_frame)
{
// Write the decoded audio frame
write_audio_frame(pToCtx, pToCtx->streams[pToCtx->nb_streams-1], frame);
}
}
av_free_packet(&pkt);
}
Decoding the audio from the input video file works fine. The problem occurs when I try to encode a mp3 frame:
static void write_audio_frame(AVFormatContext *oc, AVStream *st, AVFrame *frame)
{
AVCodecContext *enc = st->codec;
AVPacket pkt;
int got_packet = 0;
int ret = 0;
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
ret = avcodec_encode_audio2(enc, &pkt, frame, &got_packet);
if (ret < 0) {
// PROBLEM
fprintf(stderr, "Error encoding audio frame. \n");
exit(1);
}
}
I get the following console output:
[libmp3lame] inadequate AVFrame plane padding
The only happens with .flv files, the code works fine for .mp4 files. Any clue what the error message means?
Thanks
The source code containing the error message is here: http://ffmpeg.org/doxygen/trunk/libmp3lame_8c-source.html. The relevant source says:
if (frame->linesize[0] < 4 * FFALIGN(frame->nb_samples, 8)) {
av_log(avctx, AV_LOG_ERROR, "inadequate AVFrame plane padding\n");
return AVERROR(EINVAL);
}
FFALIGN is defined as
#define FFALIGN (x,a)(((x)+(a)-1)&~((a)-1))

How do I dump the buffer when encoding H264 with FFMPEG?

I'm using a c++ library to write images captured from a webcam to an libx264 encoded mp4 file.
The encoding is working properly but when it starts it writes 40 frames to the buffer. When I close the file these frames aren't flushed so about 6 seconds of video are left unwritten (cam is about 6fps).
So i'm calling:
out_size = libffmpeg::avcodec_encode_video( codecContext, data->VideoOutputBuffer,data->VideoOutputBufferSize, data->VideoFrame );
// if zero size, it means the image was buffered
if ( out_size > 0 )
{
//... write to file
}
I can't see a way of accessing the images that are left in the buffer. Any ideas?
I've got this working using the following code to flush the buffer. Seems that I was searching for the wrong term - should have been "delayed frames"...
void VideoFileWriter::Flush(void)
{
if ( data != nullptr )
{
int out_size = 0;
int ret = 0;
libffmpeg::AVCodecContext* c = data->VideoStream->codec;
/* get the delayed frames */
while (1) {
libffmpeg::AVPacket packet;
libffmpeg::av_init_packet(&packet);
out_size = libffmpeg::avcodec_encode_video(c, data->VideoOutputBuffer, data->VideoOutputBufferSize, NULL);
if (out_size < 0) {
//fprintf(stderr, "Error encoding delayed frame %d\n", out_size);
break;
}
if (out_size == 0) {
break;
}
if (c->coded_frame->pts != AV_NOPTS_VALUE) {
packet.pts = av_rescale_q(c->coded_frame->pts,
c->time_base,
data->VideoStream->time_base);
//fprintf(stderr, "Video Frame PTS: %d\n", (int)packet.pts);
} else {
//fprintf(stderr, "Video Frame PTS: not set\n");
}
if (c->coded_frame->key_frame) {
packet.flags |= AV_PKT_FLAG_KEY;
}
packet.stream_index = data->VideoStream->index;
packet.data = data->VideoOutputBuffer;
packet.size = out_size;
ret = libffmpeg::av_interleaved_write_frame( data->FormatContext, &packet );
if (ret != 0) {
//fprintf(stderr, "Error writing delayed frame %d\n", ret);
break;
}
}
libffmpeg::avcodec_flush_buffers(data->VideoStream->codec);
}
}
Here is a tutorial regarding ffmpeg with avcodec, stating that avcodec uses some internal buffers which need to be flushed. There is also some code showing how flushing of these buffers is done ("Flushing our buffers").