After many weeks trying to connect to my home cctv in my c++/Qt project using libav on my raspberry pi (armv7) I am able to correctly display my camera for a good while (at least 20 minutes) with hardly any NAL warnings in the console (but some warnings about invalid NAL unit). However, eventually I will just start to see just a grey screen and lots of NAL warnings in the console and I have to close and reopen the connection to get my camera displaying correctly. I have searched the internet many times to make sure I am understanding av_parser_parse2 correctly and I think I am.
I have pasted my 3 main functions below in the hope that somebody can tell me - is there more to decoding a h264 stream using libav than I understand currently?
Thank you.
void MainWindow::playStreamToScreen(MainWindow* app)
{
avcodec_register_all();
codec = avcodec_find_decoder(AV_CODEC_ID_H264);
if (!codec) {
qDebug() << "codec not found\n";
return;
}
c = avcodec_alloc_context3(codec);
/* if(codec->capabilities & CODEC_CAP_TRUNCATED)
c->flags |= CODEC_FLAG_TRUNCATED;
if(codec->capabilities & CODEC_FLAG2_CHUNKS)
c->flags |= CODEC_FLAG2_CHUNKS;*//* We may send incomplete frames */
picture = av_frame_alloc();
pFrameRGB=av_frame_alloc();
int numBytes=avpicture_get_size(pix_fmt_dst, myWidth, myHeight);
buffer= (uint8_t*)malloc(numBytes);
if(avpicture_fill((AVPicture *)pFrameRGB, buffer, pix_fmt_dst,
myWidth, myHeight) < 0)
{
qDebug() << "failed fill picture";
}
if (avcodec_open2(c, codec, NULL) < 0) {
qDebug() << "could not open codec\n";
return;
}
frame = 0;
if(parser == NULL)parser = av_parser_init(AV_CODEC_ID_H264);
if(parser == NULL)
{
qDebug() << "after init parser is null";
return;
}
parser->flags |= PARSER_FLAG_ONCE;
struct timeval t;
float inv_fps2;// = 4;
int ii = 0;
inv_fps2 = 1e6/9.98;//av_q2d(c->time_base);
qDebug() << "before timebase";
c->time_base = (AVRational){1,10};
c->bit_rate = 251000;
c->width = 352;
c->height = 288;
c->pix_fmt = AV_PIX_FMT_YUV420P;
if(img_convert_ctx == NULL)
{
img_convert_ctx = sws_getCachedContext ( img_convert_ctx, c->width, c->height,
AV_PIX_FMT_YUV420P, myWidth, myHeight, AV_PIX_FMT_RGB555,
SWS_LANCZOS | SWS_ACCURATE_RND , NULL, NULL, NULL );
}
running = true;
int retval;
gettimeofday(&t, 0);
}
void MainWindow::display_frame(AVCodecContext* cc, AVFrame *frame, SwsContext* img_convert_ctx, MainWindow* app)
{
sws_scale ( img_convert_ctx, frame->data, frame->linesize, 0,
288, pFrameRGB->data, pFrameRGB->linesize );
app->currentImage = QImage(pFrameRGB->data[0], app->myWidth, app->myHeight, pFrameRGB->linesize[0], QImage::Format_RGB555) .copy();// - See more at: http://www.mzan.com/article/30784549-best-simplest-way-to-display-ffmpeg-frames-in-qt5-solved.shtml#sthash.RvAFLQsc.dpuf
QImage img2 = app->currentImage.scaled(app->width(), app->height(), Qt::KeepAspectRatio);
QMetaObject::invokeMethod(app->myLabel, "setImage", Qt::QueuedConnection, Q_ARG(QImage, img2));
app->myLabel->setFixedSize(app->width(), app->height());
}
void MainWindow::socketReadyRead()
{
if(connectingToLive && i == 0)
{
qDebug() << "starting live stream";
i++;
this->playStreamToScreen(this);
}
else if(connectingToLive && i > 0)
{
QByteArray temp;
int inbuf_start = 0;
int inbuf_len = 0;
temp.append(socket->readAll());//reads from network here
inbuf_len = temp.size();
inbuf_start = 0;
int out_size;
while (inbuf_len)
{
av_init_packet(&packet2);
packet2.data = 0;
packet2.size = 0;
len = av_parser_parse2(parser, c, &packet2.data, &packet2.size,
(uint8_t*)temp.constData()+ inbuf_start, inbuf_len,
AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0);
inbuf_start += len;
inbuf_len -= len;
if(packet2.size == 0 && len >= 0)
{
break;
}
else if(len)
{
retval = avcodec_decode_video2(c, picture, &got_picture, &packet2);
if (got_picture && retval > 0)
{
display_frame(c, picture, img_convert_ctx, this);
}
}
}
}
}
Related
I'm trying to record the whole desktop stream with FFmpeg on Windows.
I found a working example here. The Problem is that some og the functions depricated. So I tried to replace them with the updated ones.
But there are some slight problems. The error "has triggered a breakpoint." occurse and also "not able to read the location."
The bigger problem is that I don't know if this is the right way to do this..
My code looks like this:
using namespace std;
/* initialize the resources*/
Recorder::Recorder()
{
av_register_all();
avcodec_register_all();
avdevice_register_all();
cout<<"\nall required functions are registered successfully";
}
/* uninitialize the resources */
Recorder::~Recorder()
{
avformat_close_input(&pAVFormatContext);
if( !pAVFormatContext )
{
cout<<"\nfile closed sucessfully";
}
else
{
cout<<"\nunable to close the file";
exit(1);
}
avformat_free_context(pAVFormatContext);
if( !pAVFormatContext )
{
cout<<"\navformat free successfully";
}
else
{
cout<<"\nunable to free avformat context";
exit(1);
}
}
/* establishing the connection between camera or screen through its respective folder */
int Recorder::openCamera()
{
value = 0;
options = NULL;
pAVFormatContext = NULL;
pAVFormatContext = avformat_alloc_context();//Allocate an AVFormatContext.
openScreen(pAVFormatContext);
/* set frame per second */
value = av_dict_set( &options,"framerate","30",0 );
if(value < 0)
{
cout<<"\nerror in setting dictionary value";
exit(1);
}
value = av_dict_set( &options, "preset", "medium", 0 );
if(value < 0)
{
cout<<"\nerror in setting preset values";
exit(1);
}
// value = avformat_find_stream_info(pAVFormatContext,NULL);
if(value < 0)
{
cout<<"\nunable to find the stream information";
exit(1);
}
VideoStreamIndx = -1;
/* find the first video stream index . Also there is an API available to do the below operations */
for(int i = 0; i < pAVFormatContext->nb_streams; i++ ) // find video stream posistion/index.
{
if( pAVFormatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO )
{
VideoStreamIndx = i;
break;
}
}
if( VideoStreamIndx == -1)
{
cout<<"\nunable to find the video stream index. (-1)";
exit(1);
}
// assign pAVFormatContext to VideoStreamIndx
pAVCodecContext = pAVFormatContext->streams[VideoStreamIndx]->codec;
pAVCodec = avcodec_find_decoder(pAVCodecContext->codec_id);
if( pAVCodec == NULL )
{
cout<<"\nunable to find the decoder";
exit(1);
}
value = avcodec_open2(pAVCodecContext , pAVCodec , NULL);//Initialize the AVCodecContext to use the given AVCodec.
if( value < 0 )
{
cout<<"\nunable to open the av codec";
exit(1);
}
}
/* initialize the video output file and its properties */
int Recorder::init_outputfile()
{
outAVFormatContext = NULL;
value = 0;
output_file = "output.mp4";
avformat_alloc_output_context2(&outAVFormatContext, NULL, NULL, output_file);
if (!outAVFormatContext)
{
cout<<"\nerror in allocating av format output context";
exit(1);
}
/* Returns the output format in the list of registered output formats which best matches the provided parameters, or returns NULL if there is no match. */
output_format = av_guess_format(NULL, output_file ,NULL);
if( !output_format )
{
cout<<"\nerror in guessing the video format. try with correct format";
exit(1);
}
video_st = avformat_new_stream(outAVFormatContext ,NULL);
if( !video_st )
{
cout<<"\nerror in creating a av format new stream";
exit(1);
}
if (codec_id == AV_CODEC_ID_H264)
{
av_opt_set(outAVCodecContext->priv_data, "preset", "slow", 0);
}
outAVCodec = avcodec_find_encoder(AV_CODEC_ID_MPEG4);
if (!outAVCodec)
{
cout << "\nerror in finding the av codecs. try again with correct codec";
exit(1);
}
outAVCodecContext = avcodec_alloc_context3(outAVCodec);
if( !outAVCodecContext )
{
cout<<"\nerror in allocating the codec contexts";
exit(1);
}
/* set property of the video file */
outAVCodecContext = video_st->codec;
outAVCodecContext->codec_id = AV_CODEC_ID_MPEG4;// AV_CODEC_ID_MPEG4; // AV_CODEC_ID_H264 // AV_CODEC_ID_MPEG1VIDEO
outAVCodecContext->codec_type = AVMEDIA_TYPE_VIDEO;
outAVCodecContext->pix_fmt = AV_PIX_FMT_YUV420P;
outAVCodecContext->bit_rate = 400000; // 2500000
outAVCodecContext->width = 1920;
outAVCodecContext->height = 1080;
outAVCodecContext->gop_size = 3;
outAVCodecContext->max_b_frames = 2;
outAVCodecContext->time_base.num = 1;
outAVCodecContext->time_base.den = 30; //15fps
/* Some container formats (like MP4) require global headers to be present
Mark the encoder so that it behaves accordingly. */
if ( outAVFormatContext->oformat->flags & AVFMT_GLOBALHEADER)
{
outAVCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
value = avcodec_open2(outAVCodecContext, outAVCodec, NULL);
if( value < 0)
{
cout<<"\nerror in opening the avcodec";
exit(1);
}
/* create empty video file */
if ( !(outAVFormatContext->flags & AVFMT_NOFILE) )
{
if( avio_open2(&outAVFormatContext->pb , output_file , AVIO_FLAG_WRITE ,NULL, NULL) < 0 )
{
cout<<"\nerror in creating the video file";
exit(1);
}
}
if(!outAVFormatContext->nb_streams)
{
cout<<"\noutput file dose not contain any stream";
exit(1);
}
/* imp: mp4 container or some advanced container file required header information*/
value = avformat_write_header(outAVFormatContext , &options);
if(value < 0)
{
cout<<"\nerror in writing the header context";
exit(1);
}
/*
// uncomment here to view the complete video file informations
cout<<"\n\nOutput file information :\n\n";
av_dump_format(outAVFormatContext , 0 ,output_file ,1);
*/
}
int Recorder::stop() {
threading = false;
demux->join();
rescale->join();
mux->join();
return 0;
}
int Recorder::start() {
initVideoThreads();
return 0;
}
int Recorder::initVideoThreads() {
demux = new thread(&Recorder::demuxVideoStream, this, pAVCodecContext, pAVFormatContext, VideoStreamIndx);
rescale = new thread(&Recorder::rescaleVideoStream, this, pAVCodecContext, outAVCodecContext);
demux = new thread(&Recorder::encodeVideoStream, this, outAVCodecContext);
return 0;
}
void Recorder::demuxVideoStream(AVCodecContext* codecContext, AVFormatContext* formatContext, int streamIndex)
{
// init packet
AVPacket* packet = (AVPacket*)av_malloc(sizeof(AVPacket));
av_init_packet(packet);
int ctr = 0;
while (threading)
{
if (av_read_frame(formatContext, packet) < 0) {
exit(1);
}
if (packet->stream_index == streamIndex)
{
int return_value; // = 0;
ctr++;
do
{
return_value = avcodec_send_packet(codecContext, packet);
} while (return_value == AVERROR(EAGAIN) && threading);
//int i = avcodec_send_packet(codecContext, packet);
if (return_value < 0 && threading) { // call Decoder
cout << "unable to decode video";
exit(1);
}
}
}
avcodec_send_packet(codecContext, NULL); // flush decoder
// return 0;
}
void Recorder::rescaleVideoStream(AVCodecContext* inCodecContext, AVCodecContext* outCodecContext)
{
bool closing = false;
AVFrame* inFrame = av_frame_alloc();
if (!inFrame)
{
cout << "\nunable to release the avframe resources";
exit(1);
}
int nbytes = av_image_get_buffer_size(outAVCodecContext->pix_fmt, outAVCodecContext->width, outAVCodecContext->height, 32);
uint8_t* video_outbuf = (uint8_t*)av_malloc(nbytes);
if (video_outbuf == NULL)
{
cout << "\nunable to allocate memory";
exit(1);
}
AVFrame* outFrame = av_frame_alloc();//Allocate an AVFrame and set its fields to default values.
if (!outFrame)
{
cout << "\nunable to release the avframe resources for outframe";
exit(1);
}
// Setup the data pointers and linesizes based on the specified image parameters and the provided array.
int value = av_image_fill_arrays(outFrame->data, outFrame->linesize, video_outbuf, AV_PIX_FMT_YUV420P, outAVCodecContext->width, outAVCodecContext->height, 1); // returns : the size in bytes required for src
if (value < 0)
{
cout << "\nerror in filling image array";
}
int ctr = 0;
while (threading || !closing) {
int value = avcodec_receive_frame(inCodecContext, inFrame);
if (value == 0) {
ctr++;
SwsContext* swsCtx_ = sws_getContext(inCodecContext->width,
inCodecContext->height,
inCodecContext->pix_fmt,
outAVCodecContext->width,
outAVCodecContext->height,
outAVCodecContext->pix_fmt,
SWS_BICUBIC, NULL, NULL, NULL);
sws_scale(swsCtx_, inFrame->data, inFrame->linesize, 0, inCodecContext->height, outFrame->data, outFrame->linesize);
int return_value;
do
{
return_value = avcodec_send_frame(outCodecContext, outFrame);
} while (return_value == AVERROR(EAGAIN) && threading);
}
closing = (value == AVERROR_EOF);
}
avcodec_send_frame(outCodecContext, NULL);
// av_free(video_outbuf);
// return 0;
}
void Recorder::encodeVideoStream(AVCodecContext* codecContext)
{
bool closing = true;
AVPacket* packet = (AVPacket*)av_malloc(sizeof(AVPacket));
av_init_packet(packet);
int ctr = 0;
while (threading || !closing) {
packet->data = NULL; // packet data will be allocated by the encoder
packet->size = 0;
ctr++;
int value = avcodec_receive_packet(codecContext, packet);
if (value == 0) {
if (packet->pts != AV_NOPTS_VALUE)
packet->pts = av_rescale_q(packet->pts, video_st->codec->time_base, video_st->time_base);
if (packet->dts != AV_NOPTS_VALUE)
packet->dts = av_rescale_q(packet->dts, video_st->codec->time_base, video_st->time_base);
//printf("Write frame %3d (size= %2d)\n", j++, packet->size / 1000);
if (av_write_frame(outAVFormatContext, packet) != 0)
{
cout << "\nerror in writing video frame";
}
}
closing = (value == AVERROR_EOF);
}
value = av_write_trailer(outAVFormatContext);
if (value < 0)
{
cout << "\nerror in writing av trailer";
exit(1);
}
// av_free(packet);
// return 0;
}
int Recorder::openScreen(AVFormatContext* pFormatCtx) {
/*
X11 video input device.
To enable this input device during configuration you need libxcb installed on your system. It will be automatically detected during configuration.
This device allows one to capture a region of an X11 display.
refer : https://www.ffmpeg.org/ffmpeg-devices.html#x11grab
*/
/* current below is for screen recording. to connect with camera use v4l2 as a input parameter for av_find_input_format */
pAVInputFormat = av_find_input_format("gdigrab");
//value = avformat_open_input(&pAVFormatContext, ":0.0+10,250", pAVInputFormat, NULL);
value = avformat_open_input(&pAVFormatContext, "desktop", pAVInputFormat, NULL);
if (value != 0)
{
cout << "\nerror in opening input device";
exit(1);
}
return 0;
}
In my Android app, I implemented the FFMpeg library, and try to use it so I can extract audio samples from an audio file on the fly.
Here is what I did (I simplified the code here so it's easier to read):
AVPacket packet;
AVCodecContext *codecContext = NULL;
AVFormatContext *formatContext;
AVFrame *frame = NULL;
SwrContext *swrContext;
int audio_stream_index = -1;
int ret;
uint8_t *localBuffer;
int FFMpegPlayer::createFFmpeg(const char *filename)
{
int ret;
AVCodec *dec;
frame = av_frame_alloc();
av_register_all();
avformat_open_input(&formatContext, filename, NULL, NULL))
avformat_find_stream_info(formatContext, NULL))
// select the audio stream
audio_stream_index = av_find_best_stream(formatContext, AVMEDIA_TYPE_AUDIO, -1, -1, &dec, 0);
// create decoding context
codecContext = avcodec_alloc_context3(dec);
avcodec_parameters_to_context(codecContext, formatContext->streams[audio_stream_index]->codecpar);
av_opt_set_int(codecContext, "refcounted_frames", 1, 0);
// init the audio decoder
avcodec_open2(codecContext, dec, NULL))
swrContext = swr_alloc();
// we assume here that the audio file is a 44100 Hz stereo audio file
localBuffer = (uint8_t *) av_malloc(44100 * 2);
swr_alloc_set_opts(swrContext, AV_CH_LAYOUT_STEREO, AV_SAMPLE_FMT_FLT, codecContext->sample_rate,
codecContext->channel_layout, codecContext->sample_fmt, codecContext->sample_rate, 0,
NULL);
swr_init(swrContext);
return 0;
}
void FFMpegPlayer::getPcmFloat(float *buffer)
{
// init :
int i, ch, dataSize;
bool extraxted = false;
float sample = 0;
// extract :
while (av_read_frame(formatContext, &packet) >= 0 && !extraxted)
{
if (packet.stream_index == audio_stream_index)
{
// send the packet with the compressed data to the decoder
ret = avcodec_send_packet(codecContext, &packet);
// read all the output frames (in general there may be any number of them
while (ret >= 0)
{
ret = avcodec_receive_frame(codecContext, frame);
if (ret == AVERROR(EAGAIN))
{
LOGW("AVERROR(EAGAIN)\n");
break;
}
else if (ret == AVERROR_EOF)
{
LOGW("AVERROR_EOF\n");
break;
}
dataSize = av_get_bytes_per_sample(codecContext->sample_fmt);
swr_convert(swrContext, &localBuffer, 44100 * 2, (const uint8_t **) frame->data, frame->nb_samples);
int a = 0;
for (i = 0; i < frame->nb_samples; i++)
{
for (ch = 0; ch < codecContext->channels; ch++)
{
memcpy(&sample, &localBuffer[(codecContext->channels * i + ch) * dataSize], dataSize);
buffer[a] = sample;
a++;
}
}
// exit extract:
extraxted = true;
}
}
}
}
Anytime I need audio samples, I call the getPcmFloat() function.
Thanks to that code, I can clearly listen to the audio file.
The problem is: I have some crackling in the sound, and I have no idea where it comes from, or how to fix it.
Does anyone know how to get the exact frames whithout glitches?
Thanks for your help.
I use FFMPEG to record video from a RTSP stream. What my code does is get current day time, create a folder with this format year/month/day/hour/minute and save the video to that folder.
When a new minute arrive, I create the new folder base on the new minute and run the record again to the new folder.
Basically It works, but the next video start time is continue the end of previous video. For example:
video1: 00:00 -> 00:55
video2: 00:56 -> ...
I hope I can set for all videos start from 00:00. Can I do that?
Here my code
ffmpeg.h
class CtFfmpeg {
public:
CtFfmpeg();
~CtFfmpeg();
void init();
int getInput();
int getOutputName(const char *filename);
int release();
int ret;
AVFormatContext *ifmt_ctx, *ofmt_ctx;
AVStream *in_stream, *out_stream;
AVPacket pkt;
const char *in_filename;
char *out_filename;
private:
int setOutput(const char *outfilename);
AVOutputFormat *ofmt;
};
ffmpeg.cpp
#include "ctffmpeg.h"
CtFfmpeg::CtFfmpeg() {
in_filename = new char [1024];
out_filename = new char [1024];
}
CtFfmpeg::~CtFfmpeg() {
delete [] in_filename;
delete [] out_filename;
}
void CtFfmpeg::init() {
avcodec_register_all();
av_register_all();
avformat_network_init();
pkt = { 0 };
av_init_packet(&pkt);
ofmt = NULL;
ifmt_ctx = NULL;
ofmt_ctx = NULL;
return;
}
int CtFfmpeg::release() {
av_write_trailer(ofmt_ctx);
avcodec_close(out_stream->codec);
// avcodec_close(in_stream->codec);
// avformat_close_input(&ifmt_ctx);
/* close output */
if (!(ofmt->flags & AVFMT_NOFILE))
avio_close(ofmt_ctx->pb);
avformat_free_context(ofmt_ctx);
av_free_packet(&pkt);
if (ret < 0 && ret != AVERROR_EOF) {
fprintf(stderr, "Error occurred\n");
return 1;
}
}
int CtFfmpeg::getInput() {
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0) {
fprintf(stderr, "Could not open input file '%s'", in_filename);
release();
}
if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0) {
fprintf(stderr, "Failed to retrieve input stream information");
release();
}
av_dump_format(ifmt_ctx, 0, in_filename, 0);
}
int CtFfmpeg::setOutput(const char *outfilename) {
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, outfilename);
if (!ofmt_ctx) {
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
release();
}
ofmt = ofmt_ctx->oformat;
for (int i = 0; i < ifmt_ctx->nb_streams; i++) {
in_stream = ifmt_ctx->streams[i];
out_stream = avformat_new_stream(ofmt_ctx, in_stream->codec->codec);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
release();
}
ret = avcodec_copy_context(out_stream->codec, in_stream->codec);
if (ret < 0) {
fprintf(stderr, "Failed to copy context from input to output stream codec context\n");
release();
}
out_stream->codec->codec_tag = 0;
if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
out_stream->codec->flags |= CODEC_FLAG_GLOBAL_HEADER;
} // for
av_dump_format(ofmt_ctx, 0, outfilename, 1);
if (!(ofmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&ofmt_ctx->pb, outfilename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", outfilename);
release();
}
}
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
release();
}
}
int CtFfmpeg::getOutputName(const char *filename){
sprintf(out_filename,filename);
setOutput(out_filename);
}
main.cpp
#include "ctfolder.h"
#include "ctffmpeg.h"
CtFfmpeg * ff;
int main(int argc, char** argv) {
if (argc < 2) {
printf("usage: %s <RTSP link> \n", argv[0]);
return 1;
}
ff = new CtFfmpeg();
ff->in_filename = argv[1]; //RTSP input link
ff->init();
ff->getInput();
string filename;
videoRecorder obj;
int start, now;
start = obj.get_current_min();
if(obj.create_folder(0755))
cout << "Cannot create folder, maybe it already exists" << endl;
else
cout << "Create folder succesfully" << endl;
int skip = 0;
while(1){
filename = obj.update_filename();
ff->getOutputName(filename.c_str());
while((now = obj.get_current_min()) == start) {
ff->ret = av_read_frame(ff->ifmt_ctx, &(ff->pkt));
skip++;
if(skip==1)
continue;
if(skip>2)
skip=2;
if (ff->ret < 0)
continue;
ff->pkt.pts = av_rescale_q_rnd(ff->pkt.pts, ff->in_stream->time_base, ff->out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
ff->pkt.dts = av_rescale_q_rnd(ff->pkt.dts, ff->in_stream->time_base, ff->out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
ff->pkt.duration = av_rescale_q(ff->pkt.duration, ff->in_stream->time_base, ff->out_stream->time_base);
ff->pkt.pos = -1;
ff->ret = av_interleaved_write_frame(ff->ofmt_ctx, &(ff->pkt));
if (ff->ret < 0) {
fprintf(stderr, "Error muxing packet\n");
continue;
}
av_free_packet(&(ff->pkt));
}
ff->release();
cout << "New minute!" << endl;
if(obj.create_folder(0755))
cout << "Cannot create folder, something's wrong" << endl;
else
cout << "Create folder succesfully" << endl;
start = now;
}
return 0;
}
You need to shift your recording packet's pts to 0.
while(<some condition>)
{
//...
int64_t pts_offset = AV_NOPTS_VALUE ;
while((now = obj.get_current_min()) == start)
{
//...
ff.pkt.pts = ...
//...
if( pts_offset == AV_NOPTS_VALUE )
{
pts_offset = ff.pkt.pts ;
}
ff.pkt.pts -= pts_offset ;
// ...
}
}
I tried to build your code and add Alexander Chernin suggestion to it but I face to muxer error!
When you decrease recording packet's pts, it's value go lower than recording packet's dts. In avcodec.h, above declaration of pts I found this comment:
pts MUST be larger or equal to dts as presentation cannot happen before decompression.
I solved this error by decreasing recording packet's dts.
ff->pkt.pts = av_rescale_q_rnd(ff->pkt.pts, ff->in_stream->ff->out_stream->(AVRounding) (AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
if (pts_offset == AV_NOPTS_VALUE) {
pts_offset = ff->pkt.pts;
}
ff->pkt.pts -= pts_offset;
ff->pkt.dts -= pts_offset;
ff->pkt.dts = av_rescale_q_rnd(ff->pkt.dts, ff->in_stream->time_base,ff->out_stream->time_base,(AVRounding) (AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
ff->pkt.duration = av_rescale_q(ff->pkt.duration,ff->in_stream->time_base,ff->out_stream->time_base);
ff->pkt.pos = -1;
I want to apply processing to a video clip with sound track, extract and process frame by frame and write result to output file. Number of frames, size of frame and speed remains unchanged in output clip. Also I want to keep the same audio track as I have in source.
I can read clip, decode frames and process then using opencv. Audio packets are also writes fine. I'm stuck on forming output video stream.
The minimal runnable code I have for now (sorry it not so short, but cant do it shorter):
extern "C" {
#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
#include "libavcodec/avcodec.h"
#include <libavutil/opt.h>
#include <libavdevice/avdevice.h>
#include <libswscale/swscale.h>
}
#include "opencv2/opencv.hpp"
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1)
#define av_frame_alloc avcodec_alloc_frame
#endif
using namespace std;
using namespace cv;
static void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt, const char *tag)
{
AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
char buf1[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_string(buf1, pkt->pts);
char buf2[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_string(buf1, pkt->dts);
char buf3[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_string(buf1, pkt->duration);
char buf4[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_time_string(buf1, pkt->pts, time_base);
char buf5[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_time_string(buf1, pkt->dts, time_base);
char buf6[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_time_string(buf1, pkt->duration, time_base);
printf("pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
buf1, buf4,
buf2, buf5,
buf3, buf6,
pkt->stream_index);
}
int main(int argc, char **argv)
{
AVOutputFormat *ofmt = NULL;
AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
AVPacket pkt;
AVFrame *pFrame = NULL;
AVFrame *pFrameRGB = NULL;
int frameFinished = 0;
pFrame = av_frame_alloc();
pFrameRGB = av_frame_alloc();
const char *in_filename, *out_filename;
int ret, i;
in_filename = "../../TestClips/Audio Video Sync Test.mp4";
out_filename = "out.mp4";
// Initialize FFMPEG
av_register_all();
// Get input file format context
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0)
{
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
// Extract streams description
if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0)
{
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
// Print detailed information about the input or output format,
// such as duration, bitrate, streams, container, programs, metadata, side data, codec and time base.
av_dump_format(ifmt_ctx, 0, in_filename, 0);
// Allocate an AVFormatContext for an output format.
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx)
{
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
// The output container format.
ofmt = ofmt_ctx->oformat;
// Allocating output streams
for (i = 0; i < ifmt_ctx->nb_streams; i++)
{
AVStream *in_stream = ifmt_ctx->streams[i];
AVStream *out_stream = avformat_new_stream(ofmt_ctx, in_stream->codec->codec);
if (!out_stream)
{
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_copy_context(out_stream->codec, in_stream->codec);
if (ret < 0)
{
fprintf(stderr, "Failed to copy context from input to output stream codec context\n");
goto end;
}
out_stream->codec->codec_tag = 0;
if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
{
out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
}
// Show output format info
av_dump_format(ofmt_ctx, 0, out_filename, 1);
// Open output file
if (!(ofmt->flags & AVFMT_NOFILE))
{
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0)
{
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
// Write output file header
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0)
{
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
// Search for input video codec info
AVCodec *in_codec = nullptr;
AVCodecContext* avctx = nullptr;
int video_stream_index = -1;
for (int i = 0; i < ifmt_ctx->nb_streams; i++)
{
if (ifmt_ctx->streams[i]->codec->coder_type == AVMEDIA_TYPE_VIDEO)
{
video_stream_index = i;
avctx = ifmt_ctx->streams[i]->codec;
in_codec = avcodec_find_decoder(avctx->codec_id);
if (!in_codec)
{
fprintf(stderr, "in codec not found\n");
exit(1);
}
break;
}
}
// Search for output video codec info
AVCodec *out_codec = nullptr;
AVCodecContext* o_avctx = nullptr;
int o_video_stream_index = -1;
for (int i = 0; i < ofmt_ctx->nb_streams; i++)
{
if (ofmt_ctx->streams[i]->codec->coder_type == AVMEDIA_TYPE_VIDEO)
{
o_video_stream_index = i;
o_avctx = ofmt_ctx->streams[i]->codec;
out_codec = avcodec_find_encoder(o_avctx->codec_id);
if (!out_codec)
{
fprintf(stderr, "out codec not found\n");
exit(1);
}
break;
}
}
// openCV pixel format
AVPixelFormat pFormat = AV_PIX_FMT_RGB24;
// Data size
int numBytes = avpicture_get_size(pFormat, avctx->width, avctx->height);
// allocate buffer
uint8_t *buffer = (uint8_t *)av_malloc(numBytes * sizeof(uint8_t));
// fill frame structure
avpicture_fill((AVPicture *)pFrameRGB, buffer, pFormat, avctx->width, avctx->height);
// frame area
int y_size = avctx->width * avctx->height;
// Open input codec
avcodec_open2(avctx, in_codec, NULL);
// Main loop
while (1)
{
AVStream *in_stream, *out_stream;
ret = av_read_frame(ifmt_ctx, &pkt);
if (ret < 0)
{
break;
}
in_stream = ifmt_ctx->streams[pkt.stream_index];
out_stream = ofmt_ctx->streams[pkt.stream_index];
log_packet(ifmt_ctx, &pkt, "in");
// copy packet
pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base, AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
log_packet(ofmt_ctx, &pkt, "out");
if (pkt.stream_index == video_stream_index)
{
avcodec_decode_video2(avctx, pFrame, &frameFinished, &pkt);
if (frameFinished)
{
struct SwsContext *img_convert_ctx;
img_convert_ctx = sws_getCachedContext(NULL,
avctx->width,
avctx->height,
avctx->pix_fmt,
avctx->width,
avctx->height,
AV_PIX_FMT_BGR24,
SWS_BICUBIC,
NULL,
NULL,
NULL);
sws_scale(img_convert_ctx,
((AVPicture*)pFrame)->data,
((AVPicture*)pFrame)->linesize,
0,
avctx->height,
((AVPicture *)pFrameRGB)->data,
((AVPicture *)pFrameRGB)->linesize);
sws_freeContext(img_convert_ctx);
// Do some image processing
cv::Mat img(pFrame->height, pFrame->width, CV_8UC3, pFrameRGB->data[0],false);
cv::GaussianBlur(img,img,Size(5,5),3);
cv::imshow("Display", img);
cv::waitKey(5);
// --------------------------------
// Transform back to initial format
// --------------------------------
img_convert_ctx = sws_getCachedContext(NULL,
avctx->width,
avctx->height,
AV_PIX_FMT_BGR24,
avctx->width,
avctx->height,
avctx->pix_fmt,
SWS_BICUBIC,
NULL,
NULL,
NULL);
sws_scale(img_convert_ctx,
((AVPicture*)pFrameRGB)->data,
((AVPicture*)pFrameRGB)->linesize,
0,
avctx->height,
((AVPicture *)pFrame)->data,
((AVPicture *)pFrame)->linesize);
// --------------------------------------------
// Something must be here
// --------------------------------------------
//
// Write fideo frame (How to write frame to output stream ?)
//
// --------------------------------------------
sws_freeContext(img_convert_ctx);
}
}
else // write sound frame
{
ret = av_interleaved_write_frame(ofmt_ctx, &pkt);
}
if (ret < 0)
{
fprintf(stderr, "Error muxing packet\n");
break;
}
// Decrease packet ref counter
av_packet_unref(&pkt);
}
av_write_trailer(ofmt_ctx);
end:
avformat_close_input(&ifmt_ctx);
// close output
if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE))
{
avio_closep(&ofmt_ctx->pb);
}
avformat_free_context(ofmt_ctx);
if (ret < 0 && ret != AVERROR_EOF)
{
char buf_err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
av_make_error_string(buf_err, AV_ERROR_MAX_STRING_SIZE, ret);
fprintf(stderr, "Error occurred: %s\n", buf_err);
return 1;
}
avcodec_close(avctx);
av_free(pFrame);
av_free(pFrameRGB);
return 0;
}
Your original code segfaults in my case. Initializing the output codec context seems to fix it. The code below works for me but I didn't test the OpenCV stuff as I don't have the lib installed.
Get the codec context:
// Search for output video codec info
AVCodec *out_codec = NULL;
AVCodecContext* o_avctx = NULL;
int o_video_stream_index = -1;
for (int i = 0; i < ofmt_ctx->nb_streams; i++)
{
if (ofmt_ctx->streams[i]->codec->coder_type == AVMEDIA_TYPE_VIDEO)
{
o_video_stream_index = i;
out_codec = avcodec_find_encoder(ofmt_ctx->streams[i]->codec->codec_id);
o_avctx = avcodec_alloc_context3(out_codec);
o_avctx->height = avctx->height;
o_avctx->width = avctx->width;
o_avctx->sample_aspect_ratio = avctx->sample_aspect_ratio;
if (out_codec->pix_fmts)
o_avctx->pix_fmt = out_codec->pix_fmts[0];
else
o_avctx->pix_fmt = avctx->pix_fmt;
o_avctx->time_base = avctx->time_base;
avcodec_open2(o_avctx, out_codec, NULL);
}
}
Encode and write:
// Main loop
while (1)
{
...
if (pkt.stream_index == video_stream_index)
{
avcodec_decode_video2(avctx, pFrame, &frameFinished, &pkt);
if (frameFinished)
{
...
// --------------------------------------------
// Something must be here
// --------------------------------------------
int got_packet = 0;
AVPacket enc_pkt = { 0 };
av_init_packet(&enc_pkt);
avcodec_encode_video2(o_avctx, &enc_pkt, pFrame, &got_packet);
av_interleaved_write_frame(ofmt_ctx, &enc_pkt);
....
}
}
you should assign processed frame's packets information to your Original packets then pass it to av_interleaved_write_frame
My goal is to proceed on audio and video of mpeg-2 file independently, and to keep synchronicity on both flows. Duration of video is about 1 or 2 minutes maximum.
First, following this post "opencv for reading videos (and do the process),ffmpeg for audio , and SDL used to play both" sounds perfect. I have done some modification on the code considering recent ffmpeg naming changes. Compilation with cmake on 64-bits machine is fine. I get an error "Unsupported codec [3]" when opening codec.
The code is following.
Second, I looking for code dealing with synchronicity on both flows.
#include "opencv/highgui.h"
#include "opencv/cv.h"
#ifndef INT64_C
#define INT64_C(c) (c ## LL)
#define UINT64_C(c) (c ## ULL)
#endif
extern "C"{
#include <SDL/SDL.h>
#include <SDL/SDL_thread.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
}
#include <iostream>
#include <stdio.h>
#include <malloc.h>
using namespace cv;
#define SDL_AUDIO_BUFFER_SIZE 1024
typedef struct PacketQueue
{
AVPacketList *first_pkt, *last_pkt;
int nb_packets;
int size;
SDL_mutex *mutex;
SDL_cond *cond;
} PacketQueue;
PacketQueue audioq;
int audioStream = -1;
int videoStream = -1;
int quit = 0;
SDL_Surface* screen = NULL;
SDL_Surface* surface = NULL;
AVFormatContext* pFormatCtx = NULL;
AVCodecContext* aCodecCtx = NULL;
AVCodecContext* pCodecCtx = NULL;
void show_frame(IplImage* img){
if (!screen){
screen = SDL_SetVideoMode(img->width, img->height, 0, 0);
if (!screen){
fprintf(stderr, "SDL: could not set video mode - exiting\n");
exit(1);
}
}
// Assuming IplImage packed as BGR 24bits
SDL_Surface* surface = SDL_CreateRGBSurfaceFrom((void*)img->imageData,
img->width,
img->height,
img->depth * img->nChannels,
img->widthStep,
0xff0000, 0x00ff00, 0x0000ff, 0
);
SDL_BlitSurface(surface, 0, screen, 0);
SDL_Flip(screen);
}
void packet_queue_init(PacketQueue *q){
memset(q, 0, sizeof(PacketQueue));
q->mutex = SDL_CreateMutex();
q->cond = SDL_CreateCond();
}
int packet_queue_put(PacketQueue *q, AVPacket *pkt){
AVPacketList *pkt1;
if (av_dup_packet(pkt) < 0){
return -1;
}
pkt1 = (AVPacketList*) av_malloc(sizeof(AVPacketList));
//pkt1 = (AVPacketList*) malloc(sizeof(AVPacketList));
if (!pkt1) return -1;
pkt1->pkt = *pkt;
pkt1->next = NULL;
SDL_LockMutex(q->mutex);
if (!q->last_pkt)
q->first_pkt = pkt1;
else
q->last_pkt->next = pkt1;
q->last_pkt = pkt1;
q->nb_packets++;
q->size += pkt1->pkt.size;
SDL_CondSignal(q->cond);
SDL_UnlockMutex(q->mutex);
return 0;
}
static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block){
AVPacketList *pkt1;
int ret;
SDL_LockMutex(q->mutex);
for (;;){
if( quit){
ret = -1;
break;
}
pkt1 = q->first_pkt;
if (pkt1){
q->first_pkt = pkt1->next;
if (!q->first_pkt)
q->last_pkt = NULL;
q->nb_packets--;
q->size -= pkt1->pkt.size;
*pkt = pkt1->pkt;
av_free(pkt1);
//free(pkt1);
ret = 1;
break;
}
else if (!block){
ret = 0;
break;
}
else{
SDL_CondWait(q->cond, q->mutex);
}
}
SDL_UnlockMutex(q->mutex);
return ret;
}
int audio_decode_frame(AVCodecContext *aCodecCtx, uint8_t *audio_buf, int buf_size){
static AVPacket pkt;
static uint8_t *audio_pkt_data = NULL;
static int audio_pkt_size = 0;
int len1, data_size;
for (;;){
while (audio_pkt_size > 0){
data_size = buf_size;
len1 = avcodec_decode_audio3(aCodecCtx, (int16_t*)audio_buf, &data_size, &pkt);
if (len1 < 0){
// if error, skip frame
audio_pkt_size = 0;
break;
}
audio_pkt_data += len1;
audio_pkt_size -= len1;
if (data_size <= 0){
// No data yet, get more frames
continue;
}
// We have data, return it and come back for more later
return data_size;
}
if (pkt.data)
av_free_packet(&pkt);
if (quit) return -1;
if (packet_queue_get(&audioq, &pkt, 1) < 0) return -1;
audio_pkt_data = pkt.data;
audio_pkt_size = pkt.size;
}
}
void audio_callback(void *userdata, Uint8 *stream, int len){
AVCodecContext *aCodecCtx = (AVCodecContext *)userdata;
int len1, audio_size;
static uint8_t audio_buf[(AVCODEC_MAX_AUDIO_FRAME_SIZE * 3) / 2];
static unsigned int audio_buf_size = 0;
static unsigned int audio_buf_index = 0;
while (len > 0){
if (audio_buf_index >= audio_buf_size){
// We have already sent all our data; get more
audio_size = audio_decode_frame(aCodecCtx, audio_buf, sizeof(audio_buf));
if(audio_size < 0){
// If error, output silence
audio_buf_size = 1024; // arbitrary?
memset(audio_buf, 0, audio_buf_size);
}
else{
audio_buf_size = audio_size;
}
audio_buf_index = 0;
}
len1 = audio_buf_size - audio_buf_index;
if (len1 > len)
len1 = len;
memcpy(stream, (uint8_t *)audio_buf + audio_buf_index, len1);
len -= len1;
stream += len1;
audio_buf_index += len1;
}
}
void setup_ffmpeg(char* filename)
{
if (avformat_open_input(&pFormatCtx, filename, NULL, NULL) != 0){
fprintf(stderr, "FFmpeg failed to open file %s!\n", filename);
exit(-1);
}
if (av_find_stream_info(pFormatCtx) < 0){
fprintf(stderr, "FFmpeg failed to retrieve stream info!\n");
exit(-1);
}
// Dump information about file onto standard error
av_dump_format(pFormatCtx, 0, filename, 0);
// Find the first video stream
int i = 0;
for (i; i < pFormatCtx->nb_streams; i++){
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && videoStream < 0){
videoStream = i;
}
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && audioStream < 0){
audioStream = i;
}
}
if (videoStream == -1){
fprintf(stderr, "No video stream found in %s!\n", filename);
exit(-1);
}
if (audioStream == -1){
fprintf(stderr, "No audio stream found in %s!\n", filename);
exit(-1);
}
// Get a pointer to the codec context for the audio stream
aCodecCtx = pFormatCtx->streams[audioStream]->codec;
// Set audio settings from codec info
SDL_AudioSpec wanted_spec;
wanted_spec.freq = aCodecCtx->sample_rate;
wanted_spec.format = AUDIO_S16SYS;
wanted_spec.channels = aCodecCtx->channels;
wanted_spec.silence = 0;
wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE;
wanted_spec.callback = audio_callback;
wanted_spec.userdata = aCodecCtx;
SDL_AudioSpec spec;
if (SDL_OpenAudio(&wanted_spec, &spec) < 0){
fprintf(stderr, "SDL_OpenAudio: %s\n", SDL_GetError());
exit(-1);
}
AVCodec* aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
if (!aCodec){
fprintf(stderr, "Unsupported codec [1]!\n");
exit(-1);
}
avcodec_open(aCodecCtx, aCodec);
// audio_st = pFormatCtx->streams[index]
packet_queue_init(&audioq);
SDL_PauseAudio(0);
// Get a pointer to the codec context for the video stream
pCodecCtx = pFormatCtx->streams[videoStream]->codec;
// Find the decoder for the video stream
AVCodec* pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if (pCodec == NULL){
fprintf(stderr, "Unsupported codec [2]!\n");
exit(-1); // Codec not found
}
// Open codec
if (avcodec_open(pCodecCtx, pCodec) < 0){
fprintf(stderr, "Unsupported codec [3]!\n");
exit(-1); // Could not open codec
}
}
int main(int argc, char* argv[])
{
if (argc < 2){
std::cout << "Usage: " << argv[0] << " <video>" << std::endl;
return -1;
}
av_register_all();
// Init SDL
if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER))
{
fprintf(stderr, "Could not initialize SDL - %s\n", SDL_GetError());
return -1;
}
// Init ffmpeg and setup some SDL stuff related to Audio
setup_ffmpeg(argv[1]);
VideoCapture cap(argv[1]);
if (!cap.isOpened()){
std::cout << "Failed to load file!" << std::endl;
return -1;
}
AVPacket packet;
while (av_read_frame(pFormatCtx, &packet) >= 0)
{
if (packet.stream_index == videoStream)
{
// Actually this is were SYNC between audio/video would happen.
// Right now I assume that every VIDEO packet contains an entire video frame, and that's not true. A video frame can be made by multiple packets!
// But for the time being, assume 1 video frame == 1 video packet,
// so instead of reading the frame through ffmpeg, I read it through OpenCV.
Mat frame;
cap >> frame; // get a new frame from camera
// do some processing on the frame, either as a Mat or as IplImage.
// For educational purposes, applying a lame grayscale conversion
IplImage ipl_frame = frame;
for (int i = 0; i < ipl_frame.width * ipl_frame.height * ipl_frame.nChannels; i += ipl_frame.nChannels)
{
ipl_frame.imageData[i] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //B
ipl_frame.imageData[i+1] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //G
ipl_frame.imageData[i+2] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //R
}
// Display it on SDL window
show_frame(&ipl_frame);
av_free_packet(&packet);
}
else if (packet.stream_index == audioStream)
{
packet_queue_put(&audioq, &packet);
}
else
{
av_free_packet(&packet);
}
SDL_Event event;
SDL_PollEvent(&event);
switch (event.type)
{
case SDL_QUIT:
SDL_FreeSurface(surface);
SDL_Quit();
break;
default:
break;
}
}
// the camera will be deinitialized automatically in VideoCapture destructor
// Close the codec
avcodec_close(pCodecCtx);
// Close the video file
av_close_input_file(pFormatCtx);
return 0;
}
I solved the "Unsupported codec" error. Remplace AVMEDIA_TYPE_VIDEO with AVMEDIA_TYPE_AUDIO in the following line :
if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && audioStream < 0){
audioStream = i;
}
Grayscale conversion is not true due to uchar opencv pixel format.
I still have the synchro not working ... any help ?
Thanks
See here for syncing:
http://dranger.com/ffmpeg/tutorial06.html
and here:
http://dranger.com/ffmpeg/tutorial07.html