H265 codec changes from hvc1 to hev1 - c++

I try to copy mp4 file with hevc video.
Source file have hvc1 codec.
Stream #0:0(und): Video: hevc (Main) (hvc1 / 0x31637668)
After copying the codec gets the value of hev1
Stream #0:0(und): Video: hevc (Main) (hev1 / 0x31766568)
How can i get hvc1?
I found that ffmpeg can do it with "-tag:v hvc1"
(https://stackoverflow.com/questions/32152090/encode-h265-to-hvc1-codec)
And how to set tag in c++ code (FFMpeg copy streams without transcode)
But it does not effect
My code:
AVOutputFormat *ofmt = NULL;
AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
AVPacket pkt;
const char *in_filename, *out_filename;
int ret, i;
int stream_index = 0;
int *stream_mapping = NULL;
int stream_mapping_size = 0;
if (argc < 3) {
printf("usage: %s input output\n"
"API example program to remux a media file with libavformat and libavcodec.\n"
"The output format is guessed according to the file extension.\n"
"\n", argv[0]);
return 1;
}
in_filename = argv[1];
out_filename = argv[2];
av_register_all();
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0) {
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0) {
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
av_dump_format(ifmt_ctx, 0, in_filename, 0);
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx) {
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
stream_mapping_size = ifmt_ctx->nb_streams;
stream_mapping = (int*)av_mallocz_array(stream_mapping_size, sizeof(*stream_mapping));
if (!stream_mapping) {
ret = AVERROR(ENOMEM);
goto end;
}
ofmt = ofmt_ctx->oformat;
for (i = 0; i < (int)ifmt_ctx->nb_streams; i++) {
AVStream *out_stream;
AVStream *in_stream = ifmt_ctx->streams[i];
AVCodecParameters *in_codecpar = in_stream->codecpar;
AVCodec *decoder = avcodec_find_decoder(in_stream->codecpar->codec_id);
AVCodec *encoder = avcodec_find_encoder(in_stream->codecpar->codec_id);
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO &&
in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
in_codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
stream_mapping[i] = -1;
continue;
}
stream_mapping[i] = stream_index++;
out_stream = avformat_new_stream(ofmt_ctx, NULL);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_parameters_copy(out_stream->codecpar, in_codecpar);
if (ret < 0) {
fprintf(stderr, "Failed to copy codec parameters\n");
goto end;
}
unsigned int tag = 0;
// for ffmpeg new api 3.x
AVCodecParameters *parameters = out_stream->codecpar;
if (av_codec_get_tag2(ofmt_ctx->oformat->codec_tag, encoder->id, &tag) == 0) {
av_log(NULL, AV_LOG_ERROR, "could not find codec tag for codec id %d, default to 0.\n", encoder->id);
}
parameters->codec_tag = tag;
out_stream->codec = avcodec_alloc_context3(encoder);
// more setting for stream->codec
avcodec_parameters_to_context(out_stream->codec, parameters);
}
av_dump_format(ofmt_ctx, 0, out_filename, 1);
if (!(ofmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
while (1) {
AVStream *in_stream, *out_stream;
ret = av_read_frame(ifmt_ctx, &pkt);
if (ret < 0)
break;
in_stream = ifmt_ctx->streams[pkt.stream_index];
if (pkt.stream_index >= stream_mapping_size ||
stream_mapping[pkt.stream_index] < 0) {
av_packet_unref(&pkt);
continue;
}
pkt.stream_index = stream_mapping[pkt.stream_index];
out_stream = ofmt_ctx->streams[pkt.stream_index];
log_packet(ifmt_ctx, &pkt, "in");
/* copy packet */
pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
log_packet(ofmt_ctx, &pkt, "out");
ret = av_interleaved_write_frame(ofmt_ctx, &pkt);
if (ret < 0) {
fprintf(stderr, "Error muxing packet\n");
break;
}
av_packet_unref(&pkt);
}
av_write_trailer(ofmt_ctx);
end:
avformat_close_input(&ifmt_ctx);
/* close output */
if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE))
avio_closep(&ofmt_ctx->pb);
avformat_free_context(ofmt_ctx);
av_freep(&stream_mapping);
if (ret < 0 && ret != AVERROR_EOF) {
fprintf(stderr, "Error occurred: %s\n", av_errTOstr(ret).c_str());
return 1;
}
After changes:
out_stream = avformat_new_stream(ofmt_ctx, NULL);
ret = avcodec_parameters_copy(out_stream->codecpar, in_codecpar);
AVCodecParameters *parameters = out_stream->codecpar;
unsigned int tag = 0;
av_codec_get_tag2(ofmt_ctx->oformat->codec_tag, encoder->id, &tag) == 0);
parameters->codec_tag = tag;
out_stream->codec = avcodec_alloc_context3(encoder);
avcodec_parameters_to_context(out_stream->codec, parameters);
if (in_codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
parameters->codec_tag = MKTAG('h', 'v', 'c', '1');
}
}
av_dump_format(ofmt_ctx, 0, out_filename, 1);
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
fprintf(stderr, "Error write header: %s\n", av_errTOstr(ret). c_str());
goto end;
}

Related

How to cut videos/audios using FFmpeg C APIs

I'm trying to write a function to cut videos/audios FFmpeg C APIs in C++. I started with the remuxing.c example from FFmpeg GitHub repository, and tried to apply the same changes mentioned in this question, but I'm getting blank screen in the beginning of the output equal to the duration that I want to cut. This is the function I came with (Differences between the function and the remuxing example noted with // <- HERE):
int cut_video(const char *in_filename, const char *out_filename, double from_seconds, double end_seconds) {
const AVOutputFormat *ofmt = NULL;
AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
AVPacket *pkt = NULL;
int ret, i;
int stream_index = 0;
int *stream_mapping = NULL;
int stream_mapping_size = 0;
pkt = av_packet_alloc();
if (!pkt) {
fprintf(stderr, "Could not allocate AVPacket\n");
return 1;
}
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0) {
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
}
if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0) {
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
}
av_dump_format(ifmt_ctx, 0, in_filename, 0);
ret = av_seek_frame(ifmt_ctx, -1, from_seconds * AV_TIME_BASE, AVSEEK_FLAG_ANY); // <- HERE
if (ret < 0) { // <- HERE
fprintf(stderr, "Error seek\n"); // <- HERE
goto end; // <- HERE
} // <- HERE
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx) {
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
stream_mapping_size = ifmt_ctx->nb_streams;
stream_mapping = (int *)av_calloc(stream_mapping_size, sizeof(*stream_mapping));
if (!stream_mapping) {
ret = AVERROR(ENOMEM);
goto end;
}
ofmt = ofmt_ctx->oformat;
for (i = 0; i < ifmt_ctx->nb_streams; i++) {
AVStream *out_stream;
AVStream *in_stream = ifmt_ctx->streams[i];
AVCodecParameters *in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO && in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
in_codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
stream_mapping[i] = -1;
continue;
}
stream_mapping[i] = stream_index++;
out_stream = avformat_new_stream(ofmt_ctx, NULL);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ret = avcodec_parameters_copy(out_stream->codecpar, in_codecpar);
if (ret < 0) {
fprintf(stderr, "Failed to copy codec parameters\n");
goto end;
}
out_stream->codecpar->codec_tag = 0;
}
av_dump_format(ofmt_ctx, 0, out_filename, 1);
if (!(ofmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
}
}
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
}
while (1) {
AVStream *in_stream, *out_stream;
ret = av_read_frame(ifmt_ctx, pkt);
if (ret < 0) break;
in_stream = ifmt_ctx->streams[pkt->stream_index];
if (pkt->stream_index >= stream_mapping_size || stream_mapping[pkt->stream_index] < 0 ||
av_q2d(in_stream->time_base) * pkt->pts > end_seconds) { // <- HERE
av_packet_unref(pkt);
continue;
}
pkt->stream_index = stream_mapping[pkt->stream_index];
out_stream = ofmt_ctx->streams[pkt->stream_index];
log_packet(ifmt_ctx, pkt, "in");
/* copy packet */
av_packet_rescale_ts(pkt, in_stream->time_base, out_stream->time_base);
pkt->pos = -1;
log_packet(ofmt_ctx, pkt, "out");
ret = av_interleaved_write_frame(ofmt_ctx, pkt);
/* pkt is now blank (av_interleaved_write_frame() takes ownership of
* its contents and resets pkt), so that no unreferencing is necessary.
* This would be different if one used av_write_frame(). */
if (ret < 0) {
fprintf(stderr, "Error muxing packet\n");
break;
}
}
av_write_trailer(ofmt_ctx);
end:
av_packet_free(&pkt);
avformat_close_input(&ifmt_ctx);
/* close output */
if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE)) avio_closep(&ofmt_ctx->pb);
avformat_free_context(ofmt_ctx);
av_freep(&stream_mapping);
if (ret < 0 && ret != AVERROR_EOF) {
fprintf(stderr, "Error occurred: %s\n", av_err2str(ret));
return 1;
}
return 0;
}
And here is how I call it:
cut_video("/Users/aliosm/Desktop/1.mp4", "/Users/aliosm/Desktop/2.mp4", 10, 40);
I searched a lot on Google and I didn't find anything useful related to this specific use-case, do you have any idea?
Finally, I was able to do that by the help from #ffmpeg channel community on Libera.Chat IRC. The final code:
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/timestamp.h>
}
/**
* #brief Print the information of the passed packet.
*
* #fn logPacket
* #param avFormatContext AVFormatContext of the given packet.
* #param avPacket AVPacket to log.
* #param tag String to tag the log output.
*/
void logPacket(const AVFormatContext *avFormatContext, const AVPacket *avPacket, const QString tag) {
AVRational *timeBase = &avFormatContext->streams[avPacket->stream_index]->time_base;
qDebug() << QString("%1: pts:%2 pts_time:%3 dts:%4 dts_time:%5 duration:%6 duration_time:%7 stream_index:%8")
.arg(tag)
.arg(av_ts2str(avPacket->pts))
.arg(av_ts2timestr(avPacket->pts, timeBase))
.arg(av_ts2str(avPacket->dts))
.arg(av_ts2timestr(avPacket->dts, timeBase))
.arg(av_ts2str(avPacket->duration))
.arg(av_ts2timestr(avPacket->duration, timeBase))
.arg(avPacket->stream_index);
}
/**
* #brief Cut a file in the given input file path based on the start and end seconds, and output the cutted file to the
* given output file path.
*
* #fn cutFile
* #param inputFilePath Input file path to be cutted.
* #param startSeconds Cutting start time in seconds.
* #param endSeconds Cutting end time in seconds.
* #param outputFilePath Output file path to write the new cutted file.
*
* #details This function will take an input file path and cut it based on the given start and end seconds. The cutted
* file will then be outputted to the given output file path.
*
* #return True if the cutting operation finished successfully, false otherwise.
*/
bool cutFile(const QString& inputFilePath, const long long& startSeconds, const long long& endSeconds,
const QString& outputFilePath) {
int operationResult;
AVPacket* avPacket = NULL;
AVFormatContext* avInputFormatContext = NULL;
AVFormatContext* avOutputFormatContext = NULL;
avPacket = av_packet_alloc();
if (!avPacket) {
qCritical("Failed to allocate AVPacket.");
return false;
}
try {
operationResult = avformat_open_input(&avInputFormatContext, inputFilePath.toStdString().c_str(), 0, 0);
if (operationResult < 0) {
throw std::runtime_error(QString("Failed to open the input file '%1'.").arg(inputFilePath).toStdString().c_str());
}
operationResult = avformat_find_stream_info(avInputFormatContext, 0);
if (operationResult < 0) {
throw std::runtime_error(QString("Failed to retrieve the input stream information.").toStdString().c_str());
}
avformat_alloc_output_context2(&avOutputFormatContext, NULL, NULL, outputFilePath.toStdString().c_str());
if (!avOutputFormatContext) {
operationResult = AVERROR_UNKNOWN;
throw std::runtime_error(QString("Failed to create the output context.").toStdString().c_str());
}
int streamIndex = 0;
int streamMapping[avInputFormatContext->nb_streams];
int streamRescaledStartSeconds[avInputFormatContext->nb_streams];
int streamRescaledEndSeconds[avInputFormatContext->nb_streams];
// Copy streams from the input file to the output file.
for (int i = 0; i < avInputFormatContext->nb_streams; i++) {
AVStream* outStream;
AVStream* inStream = avInputFormatContext->streams[i];
streamRescaledStartSeconds[i] = av_rescale_q(startSeconds * AV_TIME_BASE, AV_TIME_BASE_Q, inStream->time_base);
streamRescaledEndSeconds[i] = av_rescale_q(endSeconds * AV_TIME_BASE, AV_TIME_BASE_Q, inStream->time_base);
if (inStream->codecpar->codec_type != AVMEDIA_TYPE_AUDIO &&
inStream->codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
inStream->codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
streamMapping[i] = -1;
continue;
}
streamMapping[i] = streamIndex++;
outStream = avformat_new_stream(avOutputFormatContext, NULL);
if (!outStream) {
operationResult = AVERROR_UNKNOWN;
throw std::runtime_error(QString("Failed to allocate the output stream.").toStdString().c_str());
}
operationResult = avcodec_parameters_copy(outStream->codecpar, inStream->codecpar);
if (operationResult < 0) {
throw std::runtime_error(
QString("Failed to copy codec parameters from input stream to output stream.").toStdString().c_str());
}
outStream->codecpar->codec_tag = 0;
}
if (!(avOutputFormatContext->oformat->flags & AVFMT_NOFILE)) {
operationResult = avio_open(&avOutputFormatContext->pb, outputFilePath.toStdString().c_str(), AVIO_FLAG_WRITE);
if (operationResult < 0) {
throw std::runtime_error(
QString("Failed to open the output file '%1'.").arg(outputFilePath).toStdString().c_str());
}
}
operationResult = avformat_write_header(avOutputFormatContext, NULL);
if (operationResult < 0) {
throw std::runtime_error(QString("Error occurred when opening output file.").toStdString().c_str());
}
operationResult = avformat_seek_file(avInputFormatContext, -1, INT64_MIN, startSeconds * AV_TIME_BASE,
startSeconds * AV_TIME_BASE, 0);
if (operationResult < 0) {
throw std::runtime_error(
QString("Failed to seek the input file to the targeted start position.").toStdString().c_str());
}
while (true) {
operationResult = av_read_frame(avInputFormatContext, avPacket);
if (operationResult < 0) break;
// Skip packets from unknown streams and packets after the end cut position.
if (avPacket->stream_index >= avInputFormatContext->nb_streams || streamMapping[avPacket->stream_index] < 0 ||
avPacket->pts > streamRescaledEndSeconds[avPacket->stream_index]) {
av_packet_unref(avPacket);
continue;
}
avPacket->stream_index = streamMapping[avPacket->stream_index];
logPacket(avInputFormatContext, avPacket, "in");
// Shift the packet to its new position by subtracting the rescaled start seconds.
avPacket->pts -= streamRescaledStartSeconds[avPacket->stream_index];
avPacket->dts -= streamRescaledStartSeconds[avPacket->stream_index];
av_packet_rescale_ts(avPacket, avInputFormatContext->streams[avPacket->stream_index]->time_base,
avOutputFormatContext->streams[avPacket->stream_index]->time_base);
avPacket->pos = -1;
logPacket(avOutputFormatContext, avPacket, "out");
operationResult = av_interleaved_write_frame(avOutputFormatContext, avPacket);
if (operationResult < 0) {
throw std::runtime_error(QString("Failed to mux the packet.").toStdString().c_str());
}
}
av_write_trailer(avOutputFormatContext);
} catch (std::runtime_error e) {
qCritical("%s", e.what());
}
av_packet_free(&avPacket);
avformat_close_input(&avInputFormatContext);
if (avOutputFormatContext && !(avOutputFormatContext->oformat->flags & AVFMT_NOFILE))
avio_closep(&avOutputFormatContext->pb);
avformat_free_context(avOutputFormatContext);
if (operationResult < 0 && operationResult != AVERROR_EOF) {
qCritical("%s", QString("Error occurred: %1.").arg(av_err2str(operationResult)).toStdString().c_str());
return false;
}
return true;
}
The code is written in C++, and it is using some Qt related classes, you can remove them and use the code on plain C++ projects.
I tried my best to make it readable, I hope it is good and helpful.
Update 1: I updated the code to fix a bug in it.
Update 2: I updated the code to do some refactoring.

Adding unregistered SEI data to every frame (ffmpeg / C++ / Windows)

I am working with FFMPEG 5.2 using it with C++ in Visual Studio. What I require to do is to add a SEI Unregistered message (5) to every frame of a stream, for that, I am demuxing a MP4 container, then taking the video stream, decoding every packet to get a frame, then add SEI message to every frame, encoding and remuxing a new video stream (video only) and saving the new stream to a separate container.
To add the SEI data I use this specific code:
const char* sideDataMsg = "139FB1A9446A4DEC8CBF65B1E12D2CFDHola";;
size_t sideDataSize = sizeof(sideDataMsg);
AVBufferRef* sideDataBuffer = av_buffer_alloc(20);
sideDataBuffer->data = (uint8_t*)sideDataMsg;
AVFrameSideData* sideData = av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_SEI_UNREGISTERED, sideDataBuffer);
regarding the format of the sideDataMsg I have tried several apporaches including setting it like: "139FB1A9-446A-4DEC-8CBF65B1E12D2CFD+Hola!" which is indicated to be the required format in H.264 specs, however, even when in memory I see the SEI data is added to every frame as we observe as follows:
the resulting stream/container does not shows the expected data, below my entire code, this is mostly code taken/adapted from doc/examples folder of FFMPEG library.
BTW: I also tried setting AVCodecContext->export_side_data to different bit values (0 to FF) understanding that this can indicate the encoder to export the SEI data in every frame to be encoded but no luck.
I appreciate in advance any help from you!
// FfmpegTests.cpp : This file contains the 'main' function. Program execution begins and ends there.
//
#pragma warning(disable : 4996)
extern "C"
{
#include "libavformat/avformat.h"
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavutil/opt.h"
#include "libavutil/avutil.h"
#include "libavutil/error.h"
#include "libavfilter/buffersrc.h"
#include "libavfilter/buffersink.h"
#include "libswscale/swscale.h"
}
#pragma comment(lib, "avcodec.lib")
#pragma comment(lib, "avformat.lib")
#pragma comment(lib, "avfilter.lib")
#pragma comment(lib, "avutil.lib")
#pragma comment(lib, "swscale.lib")
#include <cstdio>
#include <iostream>
#include <chrono>
#include <thread>
static AVFormatContext* fmt_ctx;
static AVCodecContext* dec_ctx;
AVFilterGraph* filter_graph;
AVFilterContext* buffersrc_ctx;
AVFilterContext* buffersink_ctx;
static int video_stream_index = -1;
const char* filter_descr = "scale=78:24,transpose=cclock";
static int64_t last_pts = AV_NOPTS_VALUE;
// FOR SEI NAL INSERTION
const AVOutputFormat* ofmt = NULL;
AVFormatContext* ofmt_ctx = NULL;
int stream_index = 0;
int* stream_mapping = NULL;
int stream_mapping_size = 0;
int FRAMES_COUNT = 0;
const AVCodec* codec_enc;
AVCodecContext* c = NULL;
static int open_input_file(const char* filename)
{
const AVCodec* dec;
int ret;
if ((ret = avformat_open_input(&fmt_ctx, filename, NULL, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open input file\n");
return ret;
}
if ((ret = avformat_find_stream_info(fmt_ctx, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n");
return ret;
}
/* select the video stream */
ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &dec, 0);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot find a video stream in the input file\n");
return ret;
}
video_stream_index = ret;
/* create decoding context */
dec_ctx = avcodec_alloc_context3(dec);
if (!dec_ctx)
return AVERROR(ENOMEM);
avcodec_parameters_to_context(dec_ctx, fmt_ctx->streams[video_stream_index]->codecpar);
FRAMES_COUNT = fmt_ctx->streams[video_stream_index]->nb_frames;
/* init the video decoder */
if ((ret = avcodec_open2(dec_ctx, dec, NULL)) < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot open video decoder\n");
return ret;
}
return 0;
}
static int init_filters(const char* filters_descr)
{
char args[512];
int ret = 0;
const AVFilter* buffersrc = avfilter_get_by_name("buffer");
const AVFilter* buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut* outputs = avfilter_inout_alloc();
AVFilterInOut* inputs = avfilter_inout_alloc();
AVRational time_base = fmt_ctx->streams[video_stream_index]->time_base;
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE };
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
goto end;
}
/* buffer video source: the decoded frames from the decoder will be inserted here. */
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer source\n");
goto end;
}
/* buffer video sink: to terminate the filter chain. */
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sink\n");
goto end;
}
ret = av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output pixel format\n");
goto end;
}
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filters_descr,
&inputs, &outputs, NULL)) < 0)
goto end;
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
goto end;
end:
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return ret;
}
static void display_frame(const AVFrame* frame, AVRational time_base)
{
int x, y;
uint8_t* p0, * p;
int64_t delay;
if (frame->pts != AV_NOPTS_VALUE) {
if (last_pts != AV_NOPTS_VALUE) {
/* sleep roughly the right amount of time;
* usleep is in microseconds, just like AV_TIME_BASE. */
AVRational timeBaseQ;
timeBaseQ.num = 1;
timeBaseQ.den = AV_TIME_BASE;
delay = av_rescale_q(frame->pts - last_pts, time_base, timeBaseQ);
if (delay > 0 && delay < 1000000)
std::this_thread::sleep_for(std::chrono::microseconds(delay));
}
last_pts = frame->pts;
}
/* Trivial ASCII grayscale display. */
p0 = frame->data[0];
puts("\033c");
for (y = 0; y < frame->height; y++) {
p = p0;
for (x = 0; x < frame->width; x++)
putchar(" .-+#"[*(p++) / 52]);
putchar('\n');
p0 += frame->linesize[0];
}
fflush(stdout);
}
int save_frame_as_jpeg(AVCodecContext* pCodecCtx, AVFrame* pFrame, int FrameNo) {
int ret = 0;
const AVCodec* jpegCodec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
if (!jpegCodec) {
return -1;
}
AVCodecContext* jpegContext = avcodec_alloc_context3(jpegCodec);
if (!jpegContext) {
return -1;
}
jpegContext->pix_fmt = pCodecCtx->pix_fmt;
jpegContext->height = pFrame->height;
jpegContext->width = pFrame->width;
jpegContext->time_base = AVRational{ 1,10 };
jpegContext->strict_std_compliance = FF_COMPLIANCE_UNOFFICIAL;
ret = avcodec_open2(jpegContext, jpegCodec, NULL);
if (ret < 0) {
return ret;
}
FILE* JPEGFile;
char JPEGFName[256];
AVPacket packet;
packet.data = NULL;
packet.size = 0;
av_init_packet(&packet);
int gotFrame;
ret = avcodec_send_frame(jpegContext, pFrame);
if (ret < 0) {
return ret;
}
ret = avcodec_receive_packet(jpegContext, &packet);
if (ret < 0) {
return ret;
}
sprintf(JPEGFName, "c:\\folder\\dvr-%06d.jpg", FrameNo);
JPEGFile = fopen(JPEGFName, "wb");
fwrite(packet.data, 1, packet.size, JPEGFile);
fclose(JPEGFile);
av_packet_unref(&packet);
avcodec_close(jpegContext);
return 0;
}
int initialize_output_stream(AVFormatContext* input_fctx, const char* out_filename) {
int ret = 0;
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx) {
fprintf(stderr, "Could not create output context\n");
return -1;
}
stream_mapping_size = input_fctx->nb_streams;
stream_mapping = (int*)av_calloc(stream_mapping_size, sizeof(*stream_mapping));
if (!stream_mapping) {
ret = AVERROR(ENOMEM);
return -1;
}
for (int i = 0; i < input_fctx->nb_streams; i++) {
AVStream* out_stream;
AVStream* in_stream = input_fctx->streams[i];
AVCodecParameters* in_codecpar = in_stream->codecpar;
if (in_codecpar->codec_type != AVMEDIA_TYPE_AUDIO &&
in_codecpar->codec_type != AVMEDIA_TYPE_VIDEO &&
in_codecpar->codec_type != AVMEDIA_TYPE_SUBTITLE) {
stream_mapping[i] = -1;
continue;
}
stream_mapping[i] = stream_index++;
out_stream = avformat_new_stream(ofmt_ctx, NULL);
if (!out_stream) {
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
return ret;
}
ret = avcodec_parameters_copy(out_stream->codecpar, in_codecpar);
if (ret < 0) {
fprintf(stderr, "Failed to copy codec parameters\n");
return -1;
}
out_stream->codecpar->codec_tag = 0;
}
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0) {
fprintf(stderr, "Could not open output file '%s'", out_filename);
return -1;
}
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
fprintf(stderr, "Error occurred when opening output file\n");
return -1;
}
// ENCODER
codec_enc = avcodec_find_encoder_by_name("libx264");
if (!codec_enc) {
fprintf(stderr, "Codec '%s' not found\n", "libx264");
return -1;
}
c = avcodec_alloc_context3(codec_enc);
if (!c) {
fprintf(stderr, "Could not allocate video codec context\n");
exit(1);
}
c->bit_rate = dec_ctx->bit_rate;
c->width = dec_ctx->width;
c->height = dec_ctx->height;
c->time_base = dec_ctx->time_base;
c->framerate = dec_ctx->framerate;
c->gop_size = dec_ctx->gop_size;
c->max_b_frames = dec_ctx->max_b_frames;
c->pix_fmt = dec_ctx->pix_fmt;
c->time_base = AVRational{ 1,1 };
c->export_side_data = 255;
if (codec_enc->id == AV_CODEC_ID_H264)
av_opt_set(c->priv_data, "preset", "slow", 0);
ret = avcodec_open2(c, codec_enc, NULL);
if (ret < 0) {
fprintf(stderr, "Could not open codec\n");
return ret;
}
}
int add_frame_output_stream(AVFrame* frame) {
int ret;
AVPacket* pkt;
pkt = av_packet_alloc();
ret = avcodec_send_frame(c, frame);
if (ret < 0) {
fprintf(stderr, "Error sending a frame for decoding\n");
return ret;
}
while (ret >= 0) {
ret = avcodec_receive_packet(c, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return 0;
else if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
return -1;
}
pkt->stream_index = stream_mapping[pkt->stream_index];
ret = av_interleaved_write_frame(ofmt_ctx, pkt);
av_packet_unref(pkt);
}
return 0;
}
int main(int argc, char** argv)
{
AVFrame* frame;
AVFrame* filt_frame;
AVPacket* packet;
int ret, count = 0;
// FOR SEI NAL INSERTION
const char* out_filename;
if (argc < 2) {
fprintf(stderr, "Usage: %s file\n", argv[0]);
exit(1);
}
frame = av_frame_alloc();
filt_frame = av_frame_alloc();
packet = av_packet_alloc();
if (!frame || !filt_frame || !packet) {
fprintf(stderr, "Could not allocate frame or packet\n");
exit(1);
}
if ((ret = open_input_file(argv[1])) < 0)
goto end;
if ((ret = init_filters(filter_descr)) < 0)
goto end;
out_filename = argv[2];
initialize_output_stream(fmt_ctx, out_filename);
while (count < FRAMES_COUNT)
{
if ((ret = av_read_frame(fmt_ctx, packet)) < 0)
break;
if (packet->stream_index == video_stream_index) {
ret = avcodec_send_packet(dec_ctx, packet);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while sending a packet to the decoder\n");
break;
}
while (ret >= 0)
{
ret = avcodec_receive_frame(dec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
else if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while receiving a frame from the decoder\n");
goto end;
}
frame->pts = frame->best_effort_timestamp;
/* push the decoded frame into the filtergraph */
if (av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while feeding the filtergraph\n");
break;
}
/* pull filtered frames from the filtergraph */
while (1) {
ret = av_buffersink_get_frame(buffersink_ctx, filt_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
if (ret < 0)
goto end;
// display_frame(filt_frame, buffersink_ctx->inputs[0]->time_base);
av_frame_unref(filt_frame);
/* ret = save_frame_as_jpeg(dec_ctx, frame, dec_ctx->frame_number);
if (ret < 0)
goto end; */
//2. Add metadata to frames SEI
ret = av_frame_make_writable(frame);
if (ret < 0)
exit(1);
char sideDataSei[43] = "139FB1A9446A4DEC8CBF65B1E12D2CFDHola";
const char* sideDataMsg = "139FB1A9446A4DEC8CBF65B1E12D2CFDHola";
size_t sideDataSize = sizeof(sideDataMsg);
AVBufferRef* sideDataBuffer = av_buffer_alloc(20);
sideDataBuffer->data = (uint8_t*)sideDataMsg;
AVFrameSideData* sideData = av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_SEI_UNREGISTERED, sideDataBuffer);
ret = add_frame_output_stream(frame);
if (ret < 0)
goto end;
}
av_frame_unref(frame);
count++;
}
}
av_packet_unref(packet);
}
av_write_trailer(ofmt_ctx);
end:
avfilter_graph_free(&filter_graph);
avcodec_free_context(&dec_ctx);
avformat_close_input(&fmt_ctx);
av_frame_free(&frame);
av_frame_free(&filt_frame);
av_packet_free(&packet);
if (ret < 0 && ret != AVERROR_EOF) {
char errBuf[AV_ERROR_MAX_STRING_SIZE]{ 0 };
int res = av_strerror(ret, errBuf, AV_ERROR_MAX_STRING_SIZE);
fprintf(stderr, "Error: %s\n", errBuf);
exit(1);
}
exit(0);
}
Well, I came up with this solution from a friend, just had to add:
*av_opt_set_int(c->priv_data, "udu_sei", 1, 0);*
In the function initialize_output_stream after all parameters are set for AVCodecContext (c) that is being used for the output stream encoding.
Hope this helps someone!

How to use palettegen and paletteuse filters with FFmpeg in C++?

I want to create an animated gif in a QT project.
When I scale the a QImage to AV_PIX_FMT_RGB8 directly the output looks awfull with flickering artifacts and when I scale to AV_PIX_FMT_YUV420P in between the output is dithered which does not look much better.
I found out that ffmpeg is able to produce a palette file using a filter called palettegen and then convert a movie to gif using this palette.
Is there any sample c++ file out which I could use for my project or does anybody have a clue who to use these filters in code?
The filter can be describe like below
format=pix_fmts=rgb32,fps=10,scale=320:240:flags=lanczos,split [o1] [o2];[o1] palettegen [p]; [o2] fifo [o3];[o3] [p] paletteuse
You can do some fps or scale customization with the filter above.
Then we should apply this filter graph in ffmpeg.
One important note is, as the filter palettegen need the entire stream, we need to add the whole stream to the buffer source, and a end of stream to buffer source, then we can get frame from buffer sink and muxing to the output gif file.
Here is a completed example, change the micro definition in your test and be sure to use the latest version of ffmpeg.
#define INPUT_PATH "/Users/gif/Downloads/VUE_1503566813005.mp4"
#define OUTPUT_PATH "/Users/gif/Downloads/VUE.gif"
video2gif.cpp
#include <iostream>
#ifdef __cplusplus
extern "C" {
#include "libavformat/avformat.h"
#include "libavfilter/avfilter.h"
#include "libavutil/opt.h"
#include "libavfilter/buffersrc.h"
#include "libavfilter/buffersink.h"
#include "libswscale/swscale.h"
#include "libavutil/imgutils.h"
}
#endif
#define INPUT_PATH "/Users/gif/Downloads/VUE_1503566813005.mp4"
#define OUTPUT_PATH "/Users/gif/Downloads/VUE.gif"
const char *filter_descr = "format=pix_fmts=rgb32,fps=10,scale=320:240:flags=lanczos,split [o1] [o2];[o1] palettegen [p]; [o2] fifo [o3];[o3] [p] paletteuse";
static AVFormatContext* ofmt_ctx;
static AVCodecContext* o_codec_ctx;
static AVFilterGraph* filter_graph;
static AVFilterContext* buffersrc_ctx;
static AVFilterContext* buffersink_ctx;
static int init_filters(const char* filter_desc,
AVFormatContext* ifmt_ctx,
int stream_index,
AVCodecContext* dec_ctx)
{
char args[512];
int ret = 0;
AVFilter* buffersrc = avfilter_get_by_name("buffer");
AVFilter* buffersink = avfilter_get_by_name("buffersink");
AVFilterInOut* inputs = avfilter_inout_alloc();
AVFilterInOut* outputs = avfilter_inout_alloc();
AVRational time_base = ifmt_ctx->streams[stream_index]->time_base;
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_PAL8, AV_PIX_FMT_NONE };
filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
return ret;
}
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
time_base.num, time_base.den,
dec_ctx->sample_aspect_ratio.num, dec_ctx->sample_aspect_ratio.den);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in", args, nullptr, filter_graph);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "Cannot create buffer source\n");
return ret;
}
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out", nullptr, nullptr, filter_graph);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "Cannot create buffer sink\n");
return ret;
}
av_opt_set_int_list(buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "can not set output pixel format\n");
return ret;
}
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = nullptr;
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = nullptr;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filter_desc, &inputs, &outputs, nullptr)) < 0) {
av_log(nullptr, AV_LOG_ERROR, "parse filter graph error\n");
return ret;
}
if ((ret = avfilter_graph_config(filter_graph, nullptr)) < 0) {
av_log(nullptr, AV_LOG_ERROR, "config graph error\n");
}
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return 0;
}
static int init_muxer()
{
int ret = 0;
AVOutputFormat* o_fmt = av_guess_format("gif", OUTPUT_PATH, "video/gif");
ret = avformat_alloc_output_context2(&ofmt_ctx, o_fmt, "gif", OUTPUT_PATH);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "%s allocate output format\n", av_err2str(ret));
return -1;
}
AVCodec* codec = avcodec_find_encoder(AV_CODEC_ID_GIF);
if (!codec) {
return -1;
}
AVStream* stream = avformat_new_stream(ofmt_ctx, codec);
AVCodecParameters* codec_paramters = stream->codecpar;
codec_paramters->codec_tag = 0;
codec_paramters->codec_id = codec->id;
codec_paramters->codec_type = AVMEDIA_TYPE_VIDEO;
codec_paramters->width = 320;
codec_paramters->height = 240;
codec_paramters->format = AV_PIX_FMT_PAL8;
o_codec_ctx = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(o_codec_ctx, codec_paramters);
o_codec_ctx->time_base = { 1, 10 };
if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER) {
o_codec_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
ret = avcodec_open2(o_codec_ctx, codec, NULL);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "%s open output codec\n", av_err2str(ret));
return ret;
}
ret = avio_open(&ofmt_ctx->pb, OUTPUT_PATH, AVIO_FLAG_WRITE);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "%s avio open error\n", av_err2str(ret));
return ret;
}
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "%s write header\n", av_err2str(ret));
return ret;
}
av_dump_format(ofmt_ctx, -1, OUTPUT_PATH, 1);
return 0;
}
static void destroy_muxer()
{
avformat_free_context(ofmt_ctx);
avcodec_close(o_codec_ctx);
avcodec_free_context(&o_codec_ctx);
}
static void destroy_filter()
{
avfilter_free(buffersrc_ctx);
avfilter_free(buffersink_ctx);
avfilter_graph_free(&filter_graph);
}
static void muxing_one_frame(AVFrame* frame)
{
int ret = avcodec_send_frame(o_codec_ctx, frame);
AVPacket *pkt = av_packet_alloc();
av_init_packet(pkt);
while (ret >= 0) {
ret = avcodec_receive_packet(o_codec_ctx, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
av_write_frame(ofmt_ctx, pkt);
}
av_packet_unref(pkt);
}
int main(int argc, const char * argv[]) {
av_register_all();
avcodec_register_all();
avfilter_register_all();
int ret = 0;
AVFormatContext* fmt_ctx = avformat_alloc_context();
AVCodecContext* codec_ctx = nullptr;
AVCodec* codec = nullptr;
int video_index = -1;
AVPacket *pkt = av_packet_alloc();
av_init_packet(pkt);
pkt->size = 0;
pkt->data = nullptr;
ret = avformat_open_input(&fmt_ctx, INPUT_PATH, NULL, NULL);
if (ret != 0) {
std::cerr << "error open input" << std::endl;
goto die;
}
avformat_find_stream_info(fmt_ctx, NULL);
av_dump_format(fmt_ctx, -1, INPUT_PATH, 0);
ret = av_find_best_stream(fmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &codec, 0);
if (ret == AVERROR_STREAM_NOT_FOUND) {
std::cerr << "error no video stream found" << std::endl;
goto die;
}
for (unsigned i = 0; i < fmt_ctx->nb_streams; i++) {
if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
video_index = i;
if (ret == AVERROR_DECODER_NOT_FOUND) {
codec = avcodec_find_decoder(fmt_ctx->streams[i]->codecpar->codec_id);
}
break;
}
}
if (!codec) {
std::cerr << "could not find the decoder" << std::endl;
goto die;
}
codec_ctx = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(codec_ctx, fmt_ctx->streams[video_index]->codecpar);
ret = avcodec_open2(codec_ctx, codec, NULL);
if (ret < 0) {
std::cerr << "open codec error" << std::endl;
goto die;
}
if (init_muxer() < 0) {
av_log(nullptr, AV_LOG_ERROR, "could not init muxer\n");
goto die;
}
if ((ret = init_filters(filter_descr, fmt_ctx, video_index, codec_ctx)) < 0) {
av_log(nullptr, AV_LOG_ERROR, "could not init filters %s\n", av_err2str(ret));
goto die;
}
// it's time to decode
while (av_read_frame(fmt_ctx, pkt) == 0) {
if (pkt->stream_index == video_index) {
ret = avcodec_send_packet(codec_ctx, pkt);
while (ret >= 0) {
AVFrame* frame = av_frame_alloc();
ret = avcodec_receive_frame(codec_ctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
if (ret >= 0) {
// processing one frame
frame->pts = frame->best_effort_timestamp;
// palettegen need a whole stream, just add frame to buffer and don't get frame
ret = av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
if (ret < 0) {
av_log(nullptr, AV_LOG_ERROR, "error add frame to buffer source %s\n", av_err2str(ret));
}
}
av_frame_free(&frame);
}
}
}
// end of buffer
if ((ret = av_buffersrc_add_frame_flags(buffersrc_ctx, nullptr, AV_BUFFERSRC_FLAG_KEEP_REF)) >= 0) {
do {
AVFrame* filter_frame = av_frame_alloc();
ret = av_buffersink_get_frame(buffersink_ctx, filter_frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
// av_log(nullptr, AV_LOG_ERROR, "error get frame from buffer sink %s\n", av_err2str(ret));
av_frame_unref(filter_frame);
break;
}
// write the filter frame to output file
muxing_one_frame(filter_frame);
av_log(nullptr, AV_LOG_INFO, "muxing one frame\n");
av_frame_unref(filter_frame);
} while (ret >= 0);
} else {
av_log(nullptr, AV_LOG_ERROR, "error add frame to buffer source %s\n", av_err2str(ret));
}
av_packet_free(&pkt);
av_write_trailer(ofmt_ctx);
destroy_muxer();
destroy_filter();
die:
avformat_free_context(fmt_ctx);
avcodec_close(codec_ctx);
avcodec_free_context(&codec_ctx);
return 0;
}

FFMPEG:avfilter_graph_create_filter method failed when initializing filter

I want to implement the transcoding.c sample hosted on the FFMPEG website. But the avfilter_graph_create_filterfunction failed with a return code of -22 (Line 175). I only made a minor change to the source code to make it runnable in my C++ console application. I also searched it online, but couldn't find any helpful information.
Below is my code:
extern "C"
{
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavfilter/avfiltergraph.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavutil/opt.h>
#include <libavutil/pixdesc.h>
}
static AVFormatContext *ifmt_ctx;
static AVFormatContext *ofmt_ctx;
typedef struct FilteringContext {
AVFilterContext *buffersink_ctx;
AVFilterContext *buffersrc_ctx;
AVFilterGraph *filter_graph;
} FilteringContext;
static FilteringContext *filter_ctx;
static int init_filter(FilteringContext* fctx, AVCodecContext *dec_ctx,
AVCodecContext *enc_ctx, const char *filter_spec)
{
char args[512];
int ret = 0;
AVFilter *buffersrc = NULL;
AVFilter *buffersink = NULL;
AVFilterContext *buffersrc_ctx = NULL;
AVFilterContext *buffersink_ctx = NULL;
AVFilterInOut *outputs = avfilter_inout_alloc();
AVFilterInOut *inputs = avfilter_inout_alloc();
AVFilterGraph *filter_graph = avfilter_graph_alloc();
if (!outputs || !inputs || !filter_graph) {
ret = AVERROR(ENOMEM);
goto end;
}
if (dec_ctx->codec_type == AVMEDIA_TYPE_VIDEO) {
buffersrc = avfilter_get_by_name("buffer");
buffersink = avfilter_get_by_name("buffersink");
if (!buffersrc || !buffersink) {
av_log(NULL, AV_LOG_ERROR, "filtering source or sink element not found\n");
ret = AVERROR_UNKNOWN;
goto end;
}
/*sprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
dec_ctx->time_base.num, dec_ctx->time_base.den,
dec_ctx->sample_aspect_ratio.num,
dec_ctx->sample_aspect_ratio.den);*/
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer source\n");
goto end;
}
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create buffer sink\n");
goto end;
}
ret = av_opt_set_bin(buffersink_ctx, "pix_fmts",
(uint8_t*)&enc_ctx->pix_fmt, sizeof(enc_ctx->pix_fmt),
AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output pixel format\n");
goto end;
}
}
else if (dec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) {
buffersrc = avfilter_get_by_name("abuffer");
buffersink = avfilter_get_by_name("abuffersink");
if (!buffersrc || !buffersink) {
av_log(NULL, AV_LOG_ERROR, "filtering source or sink element not found\n");
ret = AVERROR_UNKNOWN;
goto end;
}
if (!dec_ctx->channel_layout)
dec_ctx->channel_layout =
av_get_default_channel_layout(dec_ctx->channels);
/*snprintf(args, sizeof(args),
"time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%"PRIx64,
dec_ctx->time_base.num, dec_ctx->time_base.den, dec_ctx->sample_rate,
av_get_sample_fmt_name(dec_ctx->sample_fmt),
dec_ctx->channel_layout);*/
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create audio buffer source\n");
goto end;
}
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out",
NULL, NULL, filter_graph);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot create audio buffer sink\n");
goto end;
}
ret = av_opt_set_bin(buffersink_ctx, "sample_fmts",
(uint8_t*)&enc_ctx->sample_fmt, sizeof(enc_ctx->sample_fmt),
AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output sample format\n");
goto end;
}
ret = av_opt_set_bin(buffersink_ctx, "channel_layouts",
(uint8_t*)&enc_ctx->channel_layout,
sizeof(enc_ctx->channel_layout), AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output channel layout\n");
goto end;
}
ret = av_opt_set_bin(buffersink_ctx, "sample_rates",
(uint8_t*)&enc_ctx->sample_rate, sizeof(enc_ctx->sample_rate),
AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Cannot set output sample rate\n");
goto end;
}
}
else {
ret = AVERROR_UNKNOWN;
goto end;
}
/* Endpoints for the filter graph. */
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx;
outputs->pad_idx = 0;
outputs->next = NULL;
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx;
inputs->pad_idx = 0;
inputs->next = NULL;
if (!outputs->name || !inputs->name) {
ret = AVERROR(ENOMEM);
goto end;
}
if ((ret = avfilter_graph_parse_ptr(filter_graph, filter_spec,
&inputs, &outputs, NULL)) < 0)
goto end;
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0)
goto end;
/* Fill FilteringContext */
fctx->buffersrc_ctx = buffersrc_ctx;
fctx->buffersink_ctx = buffersink_ctx;
fctx->filter_graph = filter_graph;
end:
avfilter_inout_free(&inputs);
avfilter_inout_free(&outputs);
return ret;
}
static int init_filters(void)
{
const char *filter_spec;
unsigned int i;
int ret;
filter_ctx = (FilteringContext *)av_malloc_array(ifmt_ctx->nb_streams, sizeof(*filter_ctx));
if (!filter_ctx)
return AVERROR(ENOMEM);
for (i = 0; i < ifmt_ctx->nb_streams; i++) {
filter_ctx[i].buffersrc_ctx = NULL;
filter_ctx[i].buffersink_ctx = NULL;
filter_ctx[i].filter_graph = NULL;
if (!(ifmt_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO
|| ifmt_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO))
continue;
if (ifmt_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
filter_spec = "null"; /* passthrough (dummy) filter for video */
else
filter_spec = "anull"; /* passthrough (dummy) filter for audio */
ret = init_filter(&filter_ctx[i], ifmt_ctx->streams[i]->codec,
ofmt_ctx->streams[i]->codec, filter_spec);
if (ret)
return ret;
}
return 0;
}
int main(int argc, char **argv)
{
int ret;
AVPacket packet = { NULL, 0 };
AVFrame *frame = NULL;
enum AVMediaType type;
unsigned int stream_index;
unsigned int i;
int got_frame;
int(*dec_func)(AVCodecContext *, AVFrame *, int *, const AVPacket *);
av_register_all();
avfilter_register_all();
if ((ret = init_filters()) < 0)
goto end;
/*...*/
system("Pause");
return ret ? 1 : 0;
}
The error code -22 or EINVAL is returned in case of one or more invalid arguments. The buffersrc filter you're about to create takes an additional argument string (args) describing the dimensions and the format of the incoming video. You comment out the creation of the string which is why the error is produced.
// this should not be comment out, or replaced with way of setting the arguments
/*sprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt,
dec_ctx->time_base.num, dec_ctx->time_base.den,
dec_ctx->sample_aspect_ratio.num,
dec_ctx->sample_aspect_ratio.den);*/
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in",
args, NULL, filter_graph);
If the filter does not know how big the incoming frames are and how to interpret the read values (e.g. are there 3 RGB channels or 3 YUV channels? How are those laid out, i.e. are they planar or packed?) then how should it read them? Right, it can't and it tries to tell you that with the error message.

Extracting the h264 part of a video file (demuxing)

I am trying to demux a video file into the video part (h264, mpeg4, h265, vp8, etc) and the audio part (mp3, aac, ac3, etc) and the subtitle part (srt) using ffmpeg in c++.
The audio part came out alright and played on all the media players I have, so also did the subtitle part. The video part however came out WITHOUT error and saved into a .h264 file but when I use ffprobe to check it or ffplay to play it, it always give the error "Invalid data found when processing input".
The code below
/* Separate a media file into audio, video and subtitle files (demuxing, complex) */
//TODO: mute error when subtitle is not present
#define __STDC_CONSTANT_MACROS
extern "C"
{
#include "libavformat/avformat.h"
}
int main()
{
//Input AVFormatContext and Output AVFormatContext
AVOutputFormat *ofmt_a = NULL, *ofmt_v = NULL, *ofmt_s = NULL;
AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx_a = NULL, *ofmt_ctx_v = NULL, *ofmt_ctx_s = NULL;
AVPacket pkt;
int ret, i;
int videoindex=-1, audioindex=-1, srtindex=-1;
int frame_index=0;
//Input file URL
const char *in_filename = "sample.mp4";
//Output file URL
const char *out_filename_v = "sample.h264";
const char *out_filename_a = "sample.mp3";
const char *out_filename_s = "sample.srt";
av_register_all();
//Input
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0) {
printf( "Could not open input file.");
goto end;
}
if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0) {
printf( "Failed to retrieve input stream information");
goto end;
}
//Output
avformat_alloc_output_context2(&ofmt_ctx_v, NULL, NULL, out_filename_v);
if (!ofmt_ctx_v) {
printf( "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ofmt_v = ofmt_ctx_v->oformat;
avformat_alloc_output_context2(&ofmt_ctx_a, NULL, NULL, out_filename_a);
if (!ofmt_ctx_a) {
printf( "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ofmt_a = ofmt_ctx_a->oformat;
avformat_alloc_output_context2(&ofmt_ctx_s, NULL, NULL, out_filename_s);
if (!ofmt_ctx_a) {
printf( "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
}
ofmt_s = ofmt_ctx_s->oformat;
for (i = 0; i < ifmt_ctx->nb_streams; i++) {
//Create output AVStream according to input AVStream
AVFormatContext *ofmt_ctx;
AVStream *in_stream = ifmt_ctx->streams[i];
AVStream *out_stream = NULL;
if(ifmt_ctx->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO){
videoindex=i;
out_stream=avformat_new_stream(ofmt_ctx_v, in_stream->codec->codec);
ofmt_ctx=ofmt_ctx_v;
}
else if(ifmt_ctx->streams[i]->codec->codec_type==AVMEDIA_TYPE_AUDIO){
audioindex=i;
out_stream=avformat_new_stream(ofmt_ctx_a, in_stream->codec->codec);
ofmt_ctx=ofmt_ctx_a;
}
else if(ifmt_ctx->streams[i]->codec->codec_type==AVMEDIA_TYPE_SUBTITLE){
srtindex=i;
out_stream=avformat_new_stream(ofmt_ctx_s, in_stream->codec->codec);
ofmt_ctx=ofmt_ctx_s;
}
else{
break;
}
if (!out_stream) {
printf( "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
}
//Copy the settings of AVCodecContext
if (avcodec_copy_context(out_stream->codec, in_stream->codec) < 0) {
printf( "Failed to copy context from input to output stream codec context\n");
goto end;
}
out_stream->codec->codec_tag = 0;
if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
out_stream->codec->flags |= CODEC_FLAG_GLOBAL_HEADER;
}
//Dump Format------------------
printf("\n==============Input Video=============\n");
av_dump_format(ifmt_ctx, 0, in_filename, 0);
printf("\n==============Output Video============\n");
av_dump_format(ofmt_ctx_v, 0, out_filename_v, 1);
printf("\n==============Output Audio============\n");
av_dump_format(ofmt_ctx_a, 0, out_filename_a, 1);
/*printf("\n==============Output Subtitle============\n");
av_dump_format(ofmt_ctx_s, 0, out_filename_s, 1);*/
printf("\n======================================\n");
//Open output file
if (!(ofmt_v->flags & AVFMT_NOFILE)) {
if (avio_open(&ofmt_ctx_v->pb, out_filename_v, AVIO_FLAG_WRITE) < 0) {
printf( "Could not open output file '%s'", out_filename_v);
goto end;
}
}
if (!(ofmt_a->flags & AVFMT_NOFILE)) {
if (avio_open(&ofmt_ctx_a->pb, out_filename_a, AVIO_FLAG_WRITE) < 0) {
printf( "Could not open output file '%s'", out_filename_a);
goto end;
}
}
if (!(ofmt_a->flags & AVFMT_NOFILE)) {
if (avio_open(&ofmt_ctx_s->pb, out_filename_s, AVIO_FLAG_WRITE) < 0) {
printf( "Could not open output file '%s'", out_filename_s);
goto end;
}
}
//Write file header
if (avformat_write_header(ofmt_ctx_v, NULL) < 0) {
printf( "Error occurred when opening video output file\n");
goto end;
}
system("pause");
if (avformat_write_header(ofmt_ctx_a, NULL) < 0) {
printf( "Error occurred when opening audio output file\n");
goto end;
}
if (avformat_write_header(ofmt_ctx_s, NULL) < 0) {
printf( "Error occurred when opening audio output file\n");
goto end;
}
AVBitStreamFilterContext* h264bsfc = av_bitstream_filter_init("h264_mp4toannexb");
while (1) {
AVFormatContext *ofmt_ctx;
AVStream *in_stream, *out_stream;
//Get an AVPacket
if (av_read_frame(ifmt_ctx, &pkt) < 0)
break;
in_stream = ifmt_ctx->streams[pkt.stream_index];
if(pkt.stream_index==videoindex){
out_stream = ofmt_ctx_v->streams[0];
ofmt_ctx=ofmt_ctx_v;
printf("Write Video Packet. size:%d\tpts:%lld\n",pkt.size,pkt.pts);
av_bitstream_filter_filter(h264bsfc, in_stream->codec, NULL, &pkt.data, &pkt.size, pkt.data, pkt.size, 0);
}else if(pkt.stream_index==audioindex){
out_stream = ofmt_ctx_a->streams[0];
ofmt_ctx=ofmt_ctx_a;
printf("Write Audio Packet. size:%d\tpts:%lld\n",pkt.size,pkt.pts);
}
else if(pkt.stream_index==srtindex){
out_stream = ofmt_ctx_s->streams[0];
ofmt_ctx=ofmt_ctx_s;
printf("Write Subtitle Packet. size:%d\tpts:%lld\n",pkt.size,pkt.pts);
}
else{
continue;
}
//Convert PTS/DTS
pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, (AVRounding)(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX));
pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
pkt.stream_index=0;
//Write
if (av_interleaved_write_frame(ofmt_ctx, &pkt) < 0) {
printf( "Error muxing packet\n");
break;
}
//printf("Write %8d frames to output file\n",frame_index);
av_free_packet(&pkt);
frame_index++;
}
av_bitstream_filter_close(h264bsfc);
//Write file trailer
av_write_trailer(ofmt_ctx_a);
av_write_trailer(ofmt_ctx_v);
av_write_trailer(ofmt_ctx_s);
end:
avformat_close_input(&ifmt_ctx);
/* close output */
if (ofmt_ctx_a && !(ofmt_a->flags & AVFMT_NOFILE))
avio_close(ofmt_ctx_a->pb);
if (ofmt_ctx_v && !(ofmt_v->flags & AVFMT_NOFILE))
avio_close(ofmt_ctx_v->pb);
if (ofmt_ctx_s && !(ofmt_s->flags & AVFMT_NOFILE))
avio_close(ofmt_ctx_s->pb);
avformat_free_context(ofmt_ctx_a);
avformat_free_context(ofmt_ctx_v);
avformat_free_context(ofmt_ctx_s);
system("pause");
if (ret < 0 && ret != AVERROR_EOF) {
printf( "Error occurred.\n");
return -1;
}
return 0;
}
EDIT 1
Screen shot of resultant h264 file
EDIT 2
I think the "error" has to do with FFMPEG's "Using AVStream.codec.time_base as a timebase hint to the muxer is deprecated. Set AVStream.time_base instead" error.
I revert to an older version of FFMPEG and with the same code, the resultant h264 file was ok!
You need to convert an H.264 bitstream from length prefixed mode to start code prefixed mode.This is required by some streaming formats, typically the MPEG-2 transport stream format ("mpegts").
Take a look at https://www.ffmpeg.org/ffmpeg-bitstream-filters.html#h264_005fmp4toannexb
Look at lines from 402 to 424 and from 842 to 843.
https://www.ffmpeg.org/doxygen/0.7/crystalhd_8c-source.html
I used it, to extract h264 from mp4.
//Use this filter on your first h264 input AVPacket
AVFormatContext *ifmt_ctx = NULL;
//...
//... //init input
//...
AVPacket *firstPacket;
//...
//... //get packet from stream
//...
uint8_t *dummy_p;
int dummy_int;
AVBitStreamFilterContext *filter = v_bitstream_filter_init("h264_mp4toannexb");
if (!filter)
{
printf("Can't open filter\n");
exit(1);
}
ret = av_bitstream_filter_filter(filter, ifmt_ctx->streams[videoindex]->codec, NULL,
&dummy_p, &dummy_int,
firstPacket->data, firstPacket->size, 0);
if( ret < 0 )
{
printf("Can't filter\n");
exit(1);
}
// use dummy_p to write to file, as first packet