I am streaming AAC audio over network, and I need to use ffmpeg to decode the stream. I have tried in local and everything works fine, but over the network I am not sure how to initialize my AVFormatContext.
I have had a look at the functions av_probe_input_buffer* and av_probe_input_format* but it doesn't look like these functions are suited for what I want to do. My AVFormatContext is always incomplete, and I cannot find an audio stream, which prevents me from getting a codec context and initializing my decoder.
The problematic piece of code looks more or less like this:
AVFormatContext *pFormatCtx = avformat_alloc_context();
AVFrame *pFrame = av_frame_alloc();
AVPacket *packet = (AVPacket *)av_malloc(sizeof(AVPacket));
av_init_packet(packet);
packet->buf = NULL;
packet->data = NULL;
pFormatCtx->flags |= AVFMT_FLAG_CUSTOM_IO;
// Read 10 packets to give ffmpeg some hint about the data
for (int i = 0; i < 10; i++) {
uint32_t packet_size;
fread(&packet_size, 1, sizeof(packet_size), f);
uint8_t *pdata = (uint8_t*)malloc(packet_size);
int len = fread(pdata, 1, packet_size, f);
AVProbeData probeData;
probeData.buf = pdata;
probeData.buf_size = packet_size - 1;
probeData.filename = "";
pFormatCtx->iformat = av_probe_input_format(&probeData, 1);
}
// This is working, no error here
if (avformat_find_stream_info(pFormatCtx, NULL) < 0){
printf("Error finding stream info!");
}
int audioStream = -1;
for (int i = 0; i < pFormatCtx->nb_streams; i++)
if(pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO){
audioStream = i;
break;
}
// I get this error, so actually no audio stream is detected
if (audioStream == -1){
printf("Didn't find a audio stream.\n");
return -1;
}
printf("Audio stream found at index %d\n", audioStream);
// I do not get here, because an audio stream is not detected.
AVCodecContext *pCodecCtx = pFormatCtx->streams[audioStream]->codec;
// This is where I want to be!
AVCodec *pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if (pCodec == NULL) {
printf("Codec not found.\n");
return -1;
}
// Open codec
if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0) {
printf("Could not open codec.\n");
return -1;
}
Related
My ultimate goal will be to split multi channel WAV files into single mono ones, after few days of experiments my plan is the sequence:
Decode audio file into a frame.
Convert interleaved frame into a planar one. (in order to separate the data buffer into multiple ones)
Grab the planar frame buffers and encode each of them into a new file.
So far I'm stuck trying to convert a wav file from interleaved to a planar one, and reprint the wav file.
edit:
I've turned on guard malloc and apparently the error is within the convert function
Here's the code:
AVCodecContext* initializeAndOpenCodecContext(AVFormatContext* formatContext, AVStream* stream){
// grab our stream, most audio files only have one anyway
const AVCodec* decoder = avcodec_find_decoder(stream->codecpar->codec_id);
if (!decoder){
std::cout << "no decoder, can't go ahead!\n";
return nullptr;
}
AVCodecContext* codecContext = avcodec_alloc_context3(decoder);
avcodec_parameters_to_context(codecContext, stream->codecpar);
int err = avcodec_open2(codecContext, decoder, nullptr);
if (err < 0){
std::cout << "couldn't open codex!\n";
}
return codecContext;
}
void initialiseResampler(SwrContext* resampler, AVFrame* inputFrame, AVFrame* outputFrame){
av_opt_set_chlayout(resampler, "in_channel_layout", &inputFrame->ch_layout, 0);
av_opt_set_chlayout(resampler, "out_channel_layout", &outputFrame->ch_layout, 0);
av_opt_set_int(resampler, "in_sample_fmt", inputFrame->format, 0);
av_opt_set_int(resampler, "out_sample_fmt", AV_SAMPLE_FMT_FLTP, 0);
av_opt_set_int(resampler, "in_sample_rate", inputFrame->sample_rate, 0);
av_opt_set_int(resampler, "out_sample_rate", outputFrame->sample_rate, 0);
}
AVFrame* initialisePlanarFrame(AVFrame* frameToInit, AVFrame* inputFrame){
//AVFrame *planar_frame = av_frame_alloc();
frameToInit->nb_samples = inputFrame->nb_samples;
frameToInit->ch_layout = inputFrame->ch_layout;
frameToInit->format = AV_SAMPLE_FMT_FLTP;
frameToInit->sample_rate = inputFrame->sample_rate;
return nullptr;
}
int main() {
AVCodecContext *codingContext= NULL;
const AVCodec *codec;
codec = avcodec_find_encoder(AV_CODEC_ID_PCM_F32LE);
codingContext = avcodec_alloc_context3(codec);
codingContext->bit_rate = 16000;
codingContext->sample_fmt = AV_SAMPLE_FMT_FLT;
codingContext->sample_rate = 48000;
codingContext->ch_layout.nb_channels = 2;
codingContext->ch_layout.order = (AVChannelOrder)0;
uint8_t **buffer_ = NULL;
AVFrame* planar_frame = NULL;
// open input
AVFormatContext* formatContext = nullptr;
int err = avformat_open_input(&formatContext, "/Users/tonytorm/Desktop/drum kits/DECAP - Drums That Knock Vol. 9/Kicks/Brash Full Metal Kick.wav", nullptr, nullptr);
if (err < 0){
fprintf(stderr, "Unable to open file!\n");
return;
}
// find audio stream
err = avformat_find_stream_info(formatContext, nullptr);
if (err > 0){
fprintf(stderr, "Unable to retrieve stream info!\n");
return;
}
int index = av_find_best_stream(formatContext, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
if (index < 0){
std::cout<< "coudn't find audio stream in this file" << '\n';
}
AVStream* stream = formatContext->streams[index];
auto fileName = "/Users/tonytorm/Desktop/newFile.wav";
FILE* newFile = fopen(fileName, "w+");
// find right codec and open it
if (auto openCodecContext = initializeAndOpenCodecContext(formatContext, stream)){
AVPacket* packet = av_packet_alloc();
AVFrame* frame = av_frame_alloc();
AVFrame* planar_frame = av_frame_alloc();
SwrContext *avr = swr_alloc(); //audio resampling context
AVChannelLayout monoChannelLayout{(AVChannelOrder)0};
monoChannelLayout.nb_channels = 2;
while (!av_read_frame(formatContext, packet)){
if (packet->stream_index != stream->index) continue; // we only care about audio
int ret = avcodec_send_packet(openCodecContext, packet);
if ( ret < 0) {
if (ret != AVERROR(EAGAIN)){ // if error is actual error not EAGAIN
std::cout << "can't do shit\n";
return;
}
}
while (int bret = avcodec_receive_frame(openCodecContext, frame) == 0){
initialisePlanarFrame(planar_frame, frame);
int buffer_size_in = av_samples_get_buffer_size(nullptr,
frame->ch_layout.nb_channels,
frame->nb_samples,
(AVSampleFormat)frame->format,
0);
int buffer_size_out = buffer_size_in/frame->ch_layout.nb_channels;
//planar_frame->linesize[0] = buffer_size_out;
int ret = av_samples_alloc(planar_frame->data,
NULL,
planar_frame->ch_layout.nb_channels,
planar_frame->nb_samples,
AV_SAMPLE_FMT_FLTP,
0);
initialiseResampler(avr, frame, planar_frame);
if (int errRet = swr_init(avr) < 0) {
fprintf(stderr, "Failed to initialize the resampling context\n");
}
if (ret < 0){
char error_message[AV_ERROR_MAX_STRING_SIZE];
av_strerror(ret, error_message, AV_ERROR_MAX_STRING_SIZE);
fprintf(stderr, "Error allocating sample buffer: %s\n", error_message);
return -1;
}
int samples_converted = swr_convert(avr,
planar_frame->data,
buffer_size_out,
(const uint8_t **)frame->data,
buffer_size_in);
if (samples_converted < 0) {
// handle error
std::cout << "error in conversion\n";
return;
}
if (avcodec_open2(codingContext, codec, NULL) < 0) {
std::cout << "can't encode!\n";
return;
}
AVPacket* nu_packet = av_packet_alloc();
while (int copy = avcodec_send_frame(codingContext, planar_frame) != 0){
if (copy == AVERROR(EAGAIN) || copy == AVERROR_EOF){
std::cout << "can't encode file\n";
return;
}
if (avcodec_receive_packet(codingContext, nu_packet) >=0){
fwrite(nu_packet->data, 4, nu_packet->size, newFile);
//av_write_frame(avc, nu_packet);
}
}
av_freep(planar_frame->data);
av_frame_unref(frame);
av_frame_unref(planar_frame);
}
// av_packet_free(&packet);
// av_packet_free(&nu_packet);
}
swr_free(&avr);
avcodec_free_context(&codingContext);
}
fclose(newFile);
}
I know i should write a header to the new wave file but for now I'm just trying to write the raw audio data. I'm getting always the same error but in different parts of the code (randomly), sometimes the code even compiles (writing the raw audio data, but filling it with some rubbish as well, i end up with a data file that is thrice the original one, sometimes i end up with a slightly smaller file - i guess the raw audio without the headers), results are basically random.
Here are some of the functions that trigger the error:
int ret = av_samples_alloc(); //(this the most common one)
swr_convert()
av_freep();
the error is:
main(64155,0x101b5d5c0) malloc: Incorrect checksum for freed object 0x106802600: probably modified after being freed.
Corrupt value: 0x0
main(64155,0x101b5d5c0) malloc: *** set a breakpoint in malloc_error_break to debug */
I would like to save thumbnails from a h264 stream that I'm turning into ffmpeg avpackets as jpegs.
I'm started with a h264 AVPacket (iframe) and decode it into an AVFrame using avcodec_send_packet/avcodec_receive_frame. Now trying to go from AVFrame and convert into AVPacket using avcodec_send_frame/avcodec_receive_packet
I can convert to png instead jpg, though I do get a the video looking like it's outputting three separate frames squeezed side by side into one. Wondering if it's one frame is R, next G, and finally B. I'm not sure, clearly I'm doing something wrong there. I figured it's possible it's the png encoder and I don't need it, so let's get jpg working first. But jpg is outputting unopenable files.
Any advice?
Here is my code:
int output_thumbnails(AVPacket* video_packet)
{
char png_file_name[max_chars_per_filename];
char thumbnail_id_char[10];
_itoa_s(thumbnail_id, thumbnail_id_char, 10);
strcpy_s(png_file_name, max_chars_per_filename, time_stamped_filepath);
strcat_s(png_file_name, max_chars_per_filename, time_stamped_filename);
strcat_s(png_file_name, max_chars_per_filename, thumbnail_id_char);
strcat_s(png_file_name, max_chars_per_filename, ".jpg");
thumbnail_id++;
int error_code = send_AVPacket_to_videocard(video_packet, av_codec_context_RTSP);
//if (error_code == AVERROR_EOF)
//{
// // error_code = videocard_to_PNG(png_file_name, av_codec_context_RTSP, av_codec_RTSP);
//}
if (error_code == AVERROR(EAGAIN)) //send packets to videocard until function returns EAGAIN
{
error_code = videocard_to_PNG(png_file_name, av_codec_context_RTSP);
//EAGAIN means that the video card buffer is ready to have the png pulled off of it
if (error_code == AVERROR_EOF)
{
// error_code = videocard_to_PNG(png_file_name, av_codec_context_RTSP, av_codec_RTSP);
}
else if (error_code == AVERROR(EAGAIN))
{
}
else
{
deal_with_av_errors(error_code, __LINE__, __FILE__);
}
}
else
{
deal_with_av_errors(error_code, __LINE__, __FILE__);
}
return 0;
}
VideoThumbnailGenerator.h:
#include "VideoThumbnailGenerator.h"
bool decoder_context_created = false;
bool encoder_context_created = false;
AVCodecContext* h264_decoder_codec_ctx;
AVCodecContext* thumbnail_encoder_codec_ctx;
int send_AVPacket_to_videocard(AVPacket* packet, AVCodecContext* codec_ctx)
{
if(!decoder_context_created)
{
AVCodec* h264_codec = avcodec_find_decoder(codec_ctx->codec_id);
h264_decoder_codec_ctx = avcodec_alloc_context3(h264_codec);
h264_decoder_codec_ctx->width = codec_ctx->width;
h264_decoder_codec_ctx->height = codec_ctx->height;
h264_decoder_codec_ctx->pix_fmt = AV_PIX_FMT_RGB24;
h264_decoder_codec_ctx->codec_type = AVMEDIA_TYPE_VIDEO;
h264_decoder_codec_ctx->skip_frame = AVDISCARD_NONINTRA;//AVDISCARD_NONREF;//AVDISCARD_NONINTRA;
h264_decoder_codec_ctx->time_base.num = 1;
h264_decoder_codec_ctx->time_base.den = 30;
h264_decoder_codec_ctx->extradata = codec_ctx->extradata;
h264_decoder_codec_ctx->extradata_size = codec_ctx->extradata_size;
int error_code = avcodec_open2(h264_decoder_codec_ctx, h264_codec, NULL);
if (!h264_codec) {
return -1;
}
if (error_code < 0)
{
return error_code;
}
decoder_context_created = true;
}
//use hardware decoding to decode video frame
int error_code = avcodec_send_packet(h264_decoder_codec_ctx, packet);
if(error_code == AVERROR(EAGAIN))
{
return AVERROR(EAGAIN);
}
if(error_code<0)
{
printf("Error: Could not send packet to video card");
return error_code;
}
return 0;
}
int videocard_to_PNG(char *png_file_path, AVCodecContext* codec_ctx)
{
if (!encoder_context_created)
{
//AVCodec* thumbnail_codec = avcodec_find_encoder(AV_CODEC_ID_PNG);
AVCodec* thumbnail_codec = avcodec_find_encoder(AV_CODEC_ID_JPEG2000);
thumbnail_encoder_codec_ctx = avcodec_alloc_context3(thumbnail_codec);
thumbnail_encoder_codec_ctx->width = 128;
thumbnail_encoder_codec_ctx->height = (int)(((float)codec_ctx->height/(float)codec_ctx->width) * 128);
thumbnail_encoder_codec_ctx->pix_fmt = AV_PIX_FMT_RGB24; //AV_PIX_FMT_YUVJ420P
thumbnail_encoder_codec_ctx->codec_type = AVMEDIA_TYPE_VIDEO;
thumbnail_encoder_codec_ctx->time_base.num = 1;
thumbnail_encoder_codec_ctx->time_base.den = 30;
bool thread_check = thumbnail_encoder_codec_ctx->thread_type & FF_THREAD_FRAME;
bool frame_threads_check = thumbnail_encoder_codec_ctx->codec->capabilities & AV_CODEC_CAP_FRAME_THREADS;
int error_code = avcodec_open2(thumbnail_encoder_codec_ctx, thumbnail_codec, NULL);
if (!thumbnail_codec) {
return -1;
}
if (error_code < 0)
{
return error_code;
}
encoder_context_created = true;
}
AVFrame* thumbnail_frame = av_frame_alloc();
AVPacket* thumbnail_packet = av_packet_alloc();
//av_init_packet(png_packet);
int error_code = avcodec_receive_frame(h264_decoder_codec_ctx, thumbnail_frame);
//check for errors everytime
//note EAGAIN errors won't get here since they won't get past while
if (error_code < 0 && error_code != AVERROR(EAGAIN))
{
printf("Error: Could not get frame from video card");
return error_code;
}
//empty buffer if there are any more frames to pull (there shouldn't be)
//while(error_code != AVERROR(EAGAIN))
//{
// //check for errors everytime
// //note EAGAIN errors won't get here since they won't get past while
// if (error_code < 0)
// {
// printf("Error: Could not get frame from video card");
// return error_code;
// }
//
// error_code = avcodec_receive_frame(h264_decoder_codec_ctx, png_frame);
//}
//now we convert back to AVPacket, this time one holding PNG info, so we can store to file
error_code = avcodec_send_frame(thumbnail_encoder_codec_ctx, thumbnail_frame);
if (error_code >= 0) {
error_code = avcodec_receive_packet(thumbnail_encoder_codec_ctx, thumbnail_packet);
FILE* out_PNG;
errno_t err = fopen_s(&out_PNG, png_file_path, "wb");
if (err == 0) {
fwrite(thumbnail_packet->data, thumbnail_packet->size, 1, out_PNG);
}
fclose(out_PNG);
}
return error_code;
}
I'm trying to use this code to get video frames from RTSP stream:
// Open the initial context variables that are needed
SwsContext *img_convert_ctx;
AVFormatContext* format_ctx = avformat_alloc_context();
AVCodecContext* codec_ctx = NULL;
int video_stream_index;
// Register everything
av_register_all();
avformat_network_init();
if (avformat_open_input(&format_ctx, argv[1],
NULL, NULL) != 0) {
return EXIT_FAILURE;
}
if (avformat_find_stream_info(format_ctx, NULL) < 0) {
return EXIT_FAILURE;
}
//search video stream
for (int i = 0; i < format_ctx->nb_streams; i++) {
if (format_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
video_stream_index = i;
}
AVPacket packet;
av_init_packet(&packet);
//open output file
AVFormatContext* output_ctx = avformat_alloc_context();
AVStream* stream = NULL;
//start reading packets from stream and write them to file
av_read_play(format_ctx); // <-- program is frozen in this API call
// Get the codec
AVCodec *codec = NULL;
cout << "avcodec_find_decoder" <<endl;
codec = avcodec_find_decoder(AV_CODEC_ID_H264);
// some more important stuff below ...
When I launch my program with MP4 file, it works correctly. If I use it with RTSP H264 stream it doesn't go blow av_read_play. Original ffmpeg binary works correctly with RTSP stream also. What may be the reason for this?
In my Android app, I implemented the FFMpeg library, and try to use it so I can extract audio samples from an audio file on the fly.
Here is what I did (I simplified the code here so it's easier to read):
AVPacket packet;
AVCodecContext *codecContext = NULL;
AVFormatContext *formatContext;
AVFrame *frame = NULL;
SwrContext *swrContext;
int audio_stream_index = -1;
int ret;
uint8_t *localBuffer;
int FFMpegPlayer::createFFmpeg(const char *filename)
{
int ret;
AVCodec *dec;
frame = av_frame_alloc();
av_register_all();
avformat_open_input(&formatContext, filename, NULL, NULL))
avformat_find_stream_info(formatContext, NULL))
// select the audio stream
audio_stream_index = av_find_best_stream(formatContext, AVMEDIA_TYPE_AUDIO, -1, -1, &dec, 0);
// create decoding context
codecContext = avcodec_alloc_context3(dec);
avcodec_parameters_to_context(codecContext, formatContext->streams[audio_stream_index]->codecpar);
av_opt_set_int(codecContext, "refcounted_frames", 1, 0);
// init the audio decoder
avcodec_open2(codecContext, dec, NULL))
swrContext = swr_alloc();
// we assume here that the audio file is a 44100 Hz stereo audio file
localBuffer = (uint8_t *) av_malloc(44100 * 2);
swr_alloc_set_opts(swrContext, AV_CH_LAYOUT_STEREO, AV_SAMPLE_FMT_FLT, codecContext->sample_rate,
codecContext->channel_layout, codecContext->sample_fmt, codecContext->sample_rate, 0,
NULL);
swr_init(swrContext);
return 0;
}
void FFMpegPlayer::getPcmFloat(float *buffer)
{
// init :
int i, ch, dataSize;
bool extraxted = false;
float sample = 0;
// extract :
while (av_read_frame(formatContext, &packet) >= 0 && !extraxted)
{
if (packet.stream_index == audio_stream_index)
{
// send the packet with the compressed data to the decoder
ret = avcodec_send_packet(codecContext, &packet);
// read all the output frames (in general there may be any number of them
while (ret >= 0)
{
ret = avcodec_receive_frame(codecContext, frame);
if (ret == AVERROR(EAGAIN))
{
LOGW("AVERROR(EAGAIN)\n");
break;
}
else if (ret == AVERROR_EOF)
{
LOGW("AVERROR_EOF\n");
break;
}
dataSize = av_get_bytes_per_sample(codecContext->sample_fmt);
swr_convert(swrContext, &localBuffer, 44100 * 2, (const uint8_t **) frame->data, frame->nb_samples);
int a = 0;
for (i = 0; i < frame->nb_samples; i++)
{
for (ch = 0; ch < codecContext->channels; ch++)
{
memcpy(&sample, &localBuffer[(codecContext->channels * i + ch) * dataSize], dataSize);
buffer[a] = sample;
a++;
}
}
// exit extract:
extraxted = true;
}
}
}
}
Anytime I need audio samples, I call the getPcmFloat() function.
Thanks to that code, I can clearly listen to the audio file.
The problem is: I have some crackling in the sound, and I have no idea where it comes from, or how to fix it.
Does anyone know how to get the exact frames whithout glitches?
Thanks for your help.
I want to apply processing to a video clip with sound track, extract and process frame by frame and write result to output file. Number of frames, size of frame and speed remains unchanged in output clip. Also I want to keep the same audio track as I have in source.
I can read clip, decode frames and process then using opencv. Audio packets are also writes fine. I'm stuck on forming output video stream.
The minimal runnable code I have for now (sorry it not so short, but cant do it shorter):
extern "C" {
#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
#include "libavcodec/avcodec.h"
#include <libavutil/opt.h>
#include <libavdevice/avdevice.h>
#include <libswscale/swscale.h>
}
#include "opencv2/opencv.hpp"
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1)
#define av_frame_alloc avcodec_alloc_frame
#endif
using namespace std;
using namespace cv;
static void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt, const char *tag)
{
AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
char buf1[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_string(buf1, pkt->pts);
char buf2[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_string(buf1, pkt->dts);
char buf3[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_string(buf1, pkt->duration);
char buf4[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_time_string(buf1, pkt->pts, time_base);
char buf5[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_time_string(buf1, pkt->dts, time_base);
char buf6[AV_TS_MAX_STRING_SIZE] = { 0 };
av_ts_make_time_string(buf1, pkt->duration, time_base);
printf("pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
buf1, buf4,
buf2, buf5,
buf3, buf6,
pkt->stream_index);
}
int main(int argc, char **argv)
{
AVOutputFormat *ofmt = NULL;
AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
AVPacket pkt;
AVFrame *pFrame = NULL;
AVFrame *pFrameRGB = NULL;
int frameFinished = 0;
pFrame = av_frame_alloc();
pFrameRGB = av_frame_alloc();
const char *in_filename, *out_filename;
int ret, i;
in_filename = "../../TestClips/Audio Video Sync Test.mp4";
out_filename = "out.mp4";
// Initialize FFMPEG
av_register_all();
// Get input file format context
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0)
{
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
// Extract streams description
if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0)
{
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
// Print detailed information about the input or output format,
// such as duration, bitrate, streams, container, programs, metadata, side data, codec and time base.
av_dump_format(ifmt_ctx, 0, in_filename, 0);
// Allocate an AVFormatContext for an output format.
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx)
{
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
// The output container format.
ofmt = ofmt_ctx->oformat;
// Allocating output streams
for (i = 0; i < ifmt_ctx->nb_streams; i++)
{
AVStream *in_stream = ifmt_ctx->streams[i];
AVStream *out_stream = avformat_new_stream(ofmt_ctx, in_stream->codec->codec);
if (!out_stream)
{
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_copy_context(out_stream->codec, in_stream->codec);
if (ret < 0)
{
fprintf(stderr, "Failed to copy context from input to output stream codec context\n");
goto end;
}
out_stream->codec->codec_tag = 0;
if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
{
out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
}
// Show output format info
av_dump_format(ofmt_ctx, 0, out_filename, 1);
// Open output file
if (!(ofmt->flags & AVFMT_NOFILE))
{
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0)
{
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
// Write output file header
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0)
{
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
// Search for input video codec info
AVCodec *in_codec = nullptr;
AVCodecContext* avctx = nullptr;
int video_stream_index = -1;
for (int i = 0; i < ifmt_ctx->nb_streams; i++)
{
if (ifmt_ctx->streams[i]->codec->coder_type == AVMEDIA_TYPE_VIDEO)
{
video_stream_index = i;
avctx = ifmt_ctx->streams[i]->codec;
in_codec = avcodec_find_decoder(avctx->codec_id);
if (!in_codec)
{
fprintf(stderr, "in codec not found\n");
exit(1);
}
break;
}
}
// Search for output video codec info
AVCodec *out_codec = nullptr;
AVCodecContext* o_avctx = nullptr;
int o_video_stream_index = -1;
for (int i = 0; i < ofmt_ctx->nb_streams; i++)
{
if (ofmt_ctx->streams[i]->codec->coder_type == AVMEDIA_TYPE_VIDEO)
{
o_video_stream_index = i;
o_avctx = ofmt_ctx->streams[i]->codec;
out_codec = avcodec_find_encoder(o_avctx->codec_id);
if (!out_codec)
{
fprintf(stderr, "out codec not found\n");
exit(1);
}
break;
}
}
// openCV pixel format
AVPixelFormat pFormat = AV_PIX_FMT_RGB24;
// Data size
int numBytes = avpicture_get_size(pFormat, avctx->width, avctx->height);
// allocate buffer
uint8_t *buffer = (uint8_t *)av_malloc(numBytes * sizeof(uint8_t));
// fill frame structure
avpicture_fill((AVPicture *)pFrameRGB, buffer, pFormat, avctx->width, avctx->height);
// frame area
int y_size = avctx->width * avctx->height;
// Open input codec
avcodec_open2(avctx, in_codec, NULL);
// Main loop
while (1)
{
AVStream *in_stream, *out_stream;
ret = av_read_frame(ifmt_ctx, &pkt);
if (ret < 0)
{
break;
}
in_stream = ifmt_ctx->streams[pkt.stream_index];
out_stream = ofmt_ctx->streams[pkt.stream_index];
log_packet(ifmt_ctx, &pkt, "in");
// copy packet
pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base, AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
log_packet(ofmt_ctx, &pkt, "out");
if (pkt.stream_index == video_stream_index)
{
avcodec_decode_video2(avctx, pFrame, &frameFinished, &pkt);
if (frameFinished)
{
struct SwsContext *img_convert_ctx;
img_convert_ctx = sws_getCachedContext(NULL,
avctx->width,
avctx->height,
avctx->pix_fmt,
avctx->width,
avctx->height,
AV_PIX_FMT_BGR24,
SWS_BICUBIC,
NULL,
NULL,
NULL);
sws_scale(img_convert_ctx,
((AVPicture*)pFrame)->data,
((AVPicture*)pFrame)->linesize,
0,
avctx->height,
((AVPicture *)pFrameRGB)->data,
((AVPicture *)pFrameRGB)->linesize);
sws_freeContext(img_convert_ctx);
// Do some image processing
cv::Mat img(pFrame->height, pFrame->width, CV_8UC3, pFrameRGB->data[0],false);
cv::GaussianBlur(img,img,Size(5,5),3);
cv::imshow("Display", img);
cv::waitKey(5);
// --------------------------------
// Transform back to initial format
// --------------------------------
img_convert_ctx = sws_getCachedContext(NULL,
avctx->width,
avctx->height,
AV_PIX_FMT_BGR24,
avctx->width,
avctx->height,
avctx->pix_fmt,
SWS_BICUBIC,
NULL,
NULL,
NULL);
sws_scale(img_convert_ctx,
((AVPicture*)pFrameRGB)->data,
((AVPicture*)pFrameRGB)->linesize,
0,
avctx->height,
((AVPicture *)pFrame)->data,
((AVPicture *)pFrame)->linesize);
// --------------------------------------------
// Something must be here
// --------------------------------------------
//
// Write fideo frame (How to write frame to output stream ?)
//
// --------------------------------------------
sws_freeContext(img_convert_ctx);
}
}
else // write sound frame
{
ret = av_interleaved_write_frame(ofmt_ctx, &pkt);
}
if (ret < 0)
{
fprintf(stderr, "Error muxing packet\n");
break;
}
// Decrease packet ref counter
av_packet_unref(&pkt);
}
av_write_trailer(ofmt_ctx);
end:
avformat_close_input(&ifmt_ctx);
// close output
if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE))
{
avio_closep(&ofmt_ctx->pb);
}
avformat_free_context(ofmt_ctx);
if (ret < 0 && ret != AVERROR_EOF)
{
char buf_err[AV_ERROR_MAX_STRING_SIZE] = { 0 };
av_make_error_string(buf_err, AV_ERROR_MAX_STRING_SIZE, ret);
fprintf(stderr, "Error occurred: %s\n", buf_err);
return 1;
}
avcodec_close(avctx);
av_free(pFrame);
av_free(pFrameRGB);
return 0;
}
Your original code segfaults in my case. Initializing the output codec context seems to fix it. The code below works for me but I didn't test the OpenCV stuff as I don't have the lib installed.
Get the codec context:
// Search for output video codec info
AVCodec *out_codec = NULL;
AVCodecContext* o_avctx = NULL;
int o_video_stream_index = -1;
for (int i = 0; i < ofmt_ctx->nb_streams; i++)
{
if (ofmt_ctx->streams[i]->codec->coder_type == AVMEDIA_TYPE_VIDEO)
{
o_video_stream_index = i;
out_codec = avcodec_find_encoder(ofmt_ctx->streams[i]->codec->codec_id);
o_avctx = avcodec_alloc_context3(out_codec);
o_avctx->height = avctx->height;
o_avctx->width = avctx->width;
o_avctx->sample_aspect_ratio = avctx->sample_aspect_ratio;
if (out_codec->pix_fmts)
o_avctx->pix_fmt = out_codec->pix_fmts[0];
else
o_avctx->pix_fmt = avctx->pix_fmt;
o_avctx->time_base = avctx->time_base;
avcodec_open2(o_avctx, out_codec, NULL);
}
}
Encode and write:
// Main loop
while (1)
{
...
if (pkt.stream_index == video_stream_index)
{
avcodec_decode_video2(avctx, pFrame, &frameFinished, &pkt);
if (frameFinished)
{
...
// --------------------------------------------
// Something must be here
// --------------------------------------------
int got_packet = 0;
AVPacket enc_pkt = { 0 };
av_init_packet(&enc_pkt);
avcodec_encode_video2(o_avctx, &enc_pkt, pFrame, &got_packet);
av_interleaved_write_frame(ofmt_ctx, &enc_pkt);
....
}
}
you should assign processed frame's packets information to your Original packets then pass it to av_interleaved_write_frame