Related
I am developing an application that record the screen and the audio from microphone. I implemented the pause function stopping video and audio thread on a condition variable, resuming them with a notify on the same condition variable. This is done in captureAudio(), in the main while. In this way works on macOS and linux, where I use avfoudation and alsa respectively, but on windows, with dshow, keep recording audio during the pause, when the thread is waiting on the condition variable. Does anybody know how can I fix this behaviour?
#include "ScreenRecorder.h"
using namespace std;
ScreenRecorder::ScreenRecorder() : pauseCapture(false), stopCapture(false), started(false), activeMenu(true) {
avcodec_register_all();
avdevice_register_all();
width = 1920;
height = 1200;
}
ScreenRecorder::~ScreenRecorder() {
if (started) {
value = av_write_trailer(outAVFormatContext);
if (value < 0) {
cerr << "Error in writing av trailer" << endl;
exit(-1);
}
avformat_close_input(&inAudioFormatContext);
if(inAudioFormatContext == nullptr){
cout << "inAudioFormatContext close successfully" << endl;
}
else{
cerr << "Error: unable to close the inAudioFormatContext" << endl;
exit(-1);
//throw "Error: unable to close the file";
}
avformat_free_context(inAudioFormatContext);
if(inAudioFormatContext == nullptr){
cout << "AudioFormat freed successfully" << endl;
}
else{
cerr << "Error: unable to free AudioFormatContext" << endl;
exit(-1);
}
avformat_close_input(&pAVFormatContext);
if (pAVFormatContext == nullptr) {
cout << "File close successfully" << endl;
}
else {
cerr << "Error: unable to close the file" << endl;
exit(-1);
//throw "Error: unable to close the file";
}
avformat_free_context(pAVFormatContext);
if (pAVFormatContext == nullptr) {
cout << "VideoFormat freed successfully" << endl;
}
else {
cerr << "Error: unable to free VideoFormatContext" << endl;
exit(-1);
}
}
}
/*==================================== VIDEO ==============================*/
int ScreenRecorder::openVideoDevice() throw() {
value = 0;
options = nullptr;
pAVFormatContext = nullptr;
pAVFormatContext = avformat_alloc_context();
string dimension = to_string(width) + "x" + to_string(height);
av_dict_set(&options, "video_size", dimension.c_str(), 0); //option to set the dimension of the screen section to record
#ifdef _WIN32
pAVInputFormat = av_find_input_format("gdigrab");
if (avformat_open_input(&pAVFormatContext, "desktop", pAVInputFormat, &options) != 0) {
cerr << "Couldn't open input stream" << endl;
exit(-1);
}
#elif defined linux
int offset_x = 0, offset_y = 0;
string url = ":0.0+" + to_string(offset_x) + "," + to_string(offset_y); //custom string to set the start point of the screen section
pAVInputFormat = av_find_input_format("x11grab");
value = avformat_open_input(&pAVFormatContext, url.c_str(), pAVInputFormat, &options);
if (value != 0) {
cerr << "Error in opening input device (video)" << endl;
exit(-1);
}
#else
value = av_dict_set(&options, "pixel_format", "0rgb", 0);
if (value < 0) {
cerr << "Error in setting pixel format" << endl;
exit(-1);
}
value = av_dict_set(&options, "video_device_index", "1", 0);
if (value < 0) {
cerr << "Error in setting video device index" << endl;
exit(-1);
}
pAVInputFormat = av_find_input_format("avfoundation");
if (avformat_open_input(&pAVFormatContext, "Capture screen 0:none", pAVInputFormat, &options) != 0) { //TODO trovare un modo per selezionare sempre lo schermo (forse "Capture screen 0")
cerr << "Error in opening input device" << endl;
exit(-1);
}
#endif
//set frame per second
value = av_dict_set(&options, "framerate", "30", 0);
if (value < 0) {
cerr << "Error in setting dictionary value (setting framerate)" << endl;
exit(-1);
}
value = av_dict_set(&options, "preset", "medium", 0);
if (value < 0) {
cerr << "Error in setting dictionary value (setting preset value)" << endl;
exit(-1);
}
/*
value = av_dict_set(&options, "vsync", "1", 0);
if(value < 0){
cerr << "Error in setting dictionary value (setting vsync value)" << endl;
exit(-1);
}
*/
value = av_dict_set(&options, "probesize", "60M", 0);
if (value < 0) {
cerr << "Error in setting probesize value" << endl;
exit(-1);
}
//get video stream infos from context
value = avformat_find_stream_info(pAVFormatContext, nullptr);
if (value < 0) {
cerr << "Error in retrieving the stream info" << endl;
exit(-1);
}
VideoStreamIndx = -1;
for (int i = 0; i < pAVFormatContext->nb_streams; i++) {
if (pAVFormatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
VideoStreamIndx = i;
break;
}
}
if (VideoStreamIndx == -1) {
cerr << "Error: unable to find video stream index" << endl;
exit(-2);
}
pAVCodecContext = pAVFormatContext->streams[VideoStreamIndx]->codec;
pAVCodec = avcodec_find_decoder(pAVCodecContext->codec_id/*params->codec_id*/);
if (pAVCodec == nullptr) {
cerr << "Error: unable to find decoder video" << endl;
exit(-1);
}
cout << "Insert height and width [h w]: "; //custom screen dimension to record
cin >> h >> w;*/
return 0;
}
/*========================================== AUDIO ============================*/
int ScreenRecorder::openAudioDevice() {
audioOptions = nullptr;
inAudioFormatContext = nullptr;
inAudioFormatContext = avformat_alloc_context();
value = av_dict_set(&audioOptions, "sample_rate", "44100", 0);
if (value < 0) {
cerr << "Error: cannot set audio sample rate" << endl;
exit(-1);
}
value = av_dict_set(&audioOptions, "async", "1", 0);
if (value < 0) {
cerr << "Error: cannot set audio sample rate" << endl;
exit(-1);
}
#if defined linux
audioInputFormat = av_find_input_format("alsa");
value = avformat_open_input(&inAudioFormatContext, "hw:0", audioInputFormat, &audioOptions);
if (value != 0) {
cerr << "Error in opening input device (audio)" << endl;
exit(-1);
}
#endif
#if defined _WIN32
audioInputFormat = av_find_input_format("dshow");
value = avformat_open_input(&inAudioFormatContext, "audio=Microfono (Realtek(R) Audio)", audioInputFormat, &audioOptions);
if (value != 0) {
cerr << "Error in opening input device (audio)" << endl;
exit(-1);
}
#endif
value = avformat_find_stream_info(inAudioFormatContext, nullptr);
if (value != 0) {
cerr << "Error: cannot find the audio stream information" << endl;
exit(-1);
}
audioStreamIndx = -1;
for (int i = 0; i < inAudioFormatContext->nb_streams; i++) {
if (inAudioFormatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
audioStreamIndx = i;
break;
}
}
if (audioStreamIndx == -1) {
cerr << "Error: unable to find audio stream index" << endl;
exit(-2);
}
}
int ScreenRecorder::initOutputFile() {
value = 0;
outAVFormatContext = nullptr;
outputAVFormat = av_guess_format(nullptr, "output.mp4", nullptr);
if (outputAVFormat == nullptr) {
cerr << "Error in guessing the video format, try with correct format" << endl;
exit(-5);
}
avformat_alloc_output_context2(&outAVFormatContext, outputAVFormat, outputAVFormat->name, "..\\media\\output.mp4");
if (outAVFormatContext == nullptr) {
cerr << "Error in allocating outAVFormatContext" << endl;
exit(-4);
}
/*===========================================================================*/
this->generateVideoStream();
this->generateAudioStream();
//create an empty video file
if (!(outAVFormatContext->flags & AVFMT_NOFILE)) {
if (avio_open2(&outAVFormatContext->pb, "..\\media\\output.mp4", AVIO_FLAG_WRITE, nullptr, nullptr) < 0) {
cerr << "Error in creating the video file" << endl;
exit(-10);
}
}
if (outAVFormatContext->nb_streams == 0) {
cerr << "Output file does not contain any stream" << endl;
exit(-11);
}
value = avformat_write_header(outAVFormatContext, &options);
if (value < 0) {
cerr << "Error in writing the header context" << endl;
exit(-12);
}
return 0;
}
/*=================================== VIDEO ==================================*/
void ScreenRecorder::generateVideoStream() {
//Generate video stream
videoSt = avformat_new_stream(outAVFormatContext, nullptr);
if (videoSt == nullptr) {
cerr << "Error in creating AVFormatStream" << endl;
exit(-6);
}
outVideoCodec = avcodec_find_encoder(AV_CODEC_ID_MPEG4); //AV_CODEC_ID_MPEG4
if (outVideoCodec == nullptr) {
cerr << "Error in finding the AVCodec, try again with the correct codec" << endl;
exit(-8);
}
avcodec_alloc_context3(outAVCodec)
outVideoCodecContext = avcodec_alloc_context3(outVideoCodec);
if (outVideoCodecContext == nullptr) {
cerr << "Error in allocating the codec context" << endl;
exit(-7);
}
//set properties of the video file (stream)
outVideoCodecContext = videoSt->codec;
outVideoCodecContext->codec_id = AV_CODEC_ID_MPEG4;
outVideoCodecContext->codec_type = AVMEDIA_TYPE_VIDEO;
outVideoCodecContext->pix_fmt = AV_PIX_FMT_YUV420P;
outVideoCodecContext->bit_rate = 10000000;
outVideoCodecContext->width = width;
outVideoCodecContext->height = height;
outVideoCodecContext->gop_size = 10;
outVideoCodecContext->global_quality = 500;
outVideoCodecContext->max_b_frames = 2;
outVideoCodecContext->time_base.num = 1;
outVideoCodecContext->time_base.den = 30;
outVideoCodecContext->bit_rate_tolerance = 400000;
if (outVideoCodecContext->codec_id == AV_CODEC_ID_H264) {
av_opt_set(outVideoCodecContext->priv_data, "preset", "slow", 0);
}
if (outAVFormatContext->oformat->flags & AVFMT_GLOBALHEADER) {
outVideoCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
value = avcodec_open2(outVideoCodecContext, outVideoCodec, nullptr);
if (value < 0) {
cerr << "Error in opening the AVCodec" << endl;
exit(-9);
}
outVideoStreamIndex = -1;
for (int i = 0; i < outAVFormatContext->nb_streams; i++) {
if (outAVFormatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_UNKNOWN) {
outVideoStreamIndex = i;
}
}
if (outVideoStreamIndex < 0) {
cerr << "Error: cannot find a free stream index for video output" << endl;
exit(-1);
}
avcodec_parameters_from_context(outAVFormatContext->streams[outVideoStreamIndex]->codecpar, outVideoCodecContext);
}
/*=============================== AUDIO ==================================*/
void ScreenRecorder::generateAudioStream() {
AVCodecParameters* params = inAudioFormatContext->streams[audioStreamIndx]->codecpar;
inAudioCodec = avcodec_find_decoder(params->codec_id);
if (inAudioCodec == nullptr) {
cerr << "Error: cannot find the audio decoder" << endl;
exit(-1);
}
inAudioCodecContext = avcodec_alloc_context3(inAudioCodec);
if (avcodec_parameters_to_context(inAudioCodecContext, params) < 0) {
cout << "Cannot create codec context for audio input" << endl;
}
value = avcodec_open2(inAudioCodecContext, inAudioCodec, nullptr);
if (value < 0) {
cerr << "Error: cannot open the input audio codec" << endl;
exit(-1);
}
//Generate audio stream
outAudioCodecContext = nullptr;
outAudioCodec = nullptr;
int i;
AVStream* audio_st = avformat_new_stream(outAVFormatContext, nullptr);
if (audio_st == nullptr) {
cerr << "Error: cannot create audio stream" << endl;
exit(1);
}
outAudioCodec = avcodec_find_encoder(AV_CODEC_ID_AAC);
if (outAudioCodec == nullptr) {
cerr << "Error: cannot find requested encoder" << endl;
exit(1);
}
outAudioCodecContext = avcodec_alloc_context3(outAudioCodec);
if (outAudioCodecContext == nullptr) {
cerr << "Error: cannot create related VideoCodecContext" << endl;
exit(1);
}
if ((outAudioCodec)->supported_samplerates) {
outAudioCodecContext->sample_rate = (outAudioCodec)->supported_samplerates[0];
for (i = 0; (outAudioCodec)->supported_samplerates[i]; i++) {
if ((outAudioCodec)->supported_samplerates[i] == inAudioCodecContext->sample_rate)
outAudioCodecContext->sample_rate = inAudioCodecContext->sample_rate;
}
}
outAudioCodecContext->codec_id = AV_CODEC_ID_AAC;
outAudioCodecContext->sample_fmt = (outAudioCodec)->sample_fmts ? (outAudioCodec)->sample_fmts[0] : AV_SAMPLE_FMT_FLTP;
outAudioCodecContext->channels = inAudioCodecContext->channels;
outAudioCodecContext->channel_layout = av_get_default_channel_layout(outAudioCodecContext->channels);
outAudioCodecContext->bit_rate = 96000;
outAudioCodecContext->time_base = { 1, inAudioCodecContext->sample_rate };
outAudioCodecContext->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL;
if ((outAVFormatContext)->oformat->flags & AVFMT_GLOBALHEADER) {
outAudioCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
if (avcodec_open2(outAudioCodecContext, outAudioCodec, nullptr) < 0) {
cerr << "error in opening the avcodec" << endl;
exit(1);
}
//find a free stream index
outAudioStreamIndex = -1;
for (i = 0; i < outAVFormatContext->nb_streams; i++) {
if (outAVFormatContext->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_UNKNOWN) {
outAudioStreamIndex = i;
}
}
if (outAudioStreamIndex < 0) {
cerr << "Error: cannot find a free stream for audio on the output" << endl;
exit(1);
}
avcodec_parameters_from_context(outAVFormatContext->streams[outAudioStreamIndex]->codecpar, outAudioCodecContext);
}
int ScreenRecorder::init_fifo()
{
/* Create the FIFO buffer based on the specified output sample format. */
if (!(fifo = av_audio_fifo_alloc(outAudioCodecContext->sample_fmt,
outAudioCodecContext->channels, 1))) {
fprintf(stderr, "Could not allocate FIFO\n");
return AVERROR(ENOMEM);
}
return 0;
}
int ScreenRecorder::add_samples_to_fifo(uint8_t** converted_input_samples, const int frame_size) {
int error;
/* Make the FIFO as large as it needs to be to hold both,
* the old and the new samples. */
if ((error = av_audio_fifo_realloc(fifo, av_audio_fifo_size(fifo) + frame_size)) < 0) {
fprintf(stderr, "Could not reallocate FIFO\n");
return error;
}
/* Store the new samples in the FIFO buffer. */
if (av_audio_fifo_write(fifo, (void**)converted_input_samples, frame_size) < frame_size) {
fprintf(stderr, "Could not write data to FIFO\n");
return AVERROR_EXIT;
}
return 0;
}
int ScreenRecorder::initConvertedSamples(uint8_t*** converted_input_samples,
AVCodecContext* output_codec_context,
int frame_size) {
int error;
/* Allocate as many pointers as there are audio channels.
* Each pointer will later point to the audio samples of the corresponding
* channels (although it may be NULL for interleaved formats).
*/
if (!(*converted_input_samples = (uint8_t**)calloc(output_codec_context->channels,
sizeof(**converted_input_samples)))) {
fprintf(stderr, "Could not allocate converted input sample pointers\n");
return AVERROR(ENOMEM);
}
/* Allocate memory for the samples of all channels in one consecutive
* block for convenience. */
if (av_samples_alloc(*converted_input_samples, nullptr,
output_codec_context->channels,
frame_size,
output_codec_context->sample_fmt, 0) < 0) {
exit(1);
}
return 0;
}
static int64_t pts = 0;
void ScreenRecorder::captureAudio() {
int ret;
AVPacket* inPacket, * outPacket;
AVFrame* rawFrame, * scaledFrame;
uint8_t** resampledData;
init_fifo();
//allocate space for a packet
inPacket = (AVPacket*)av_malloc(sizeof(AVPacket));
if (!inPacket) {
cerr << "Cannot allocate an AVPacket for encoded video" << endl;
exit(1);
}
av_init_packet(inPacket);
//allocate space for a packet
rawFrame = av_frame_alloc();
if (!rawFrame) {
cerr << "Cannot allocate an AVPacket for encoded video" << endl;
exit(1);
}
scaledFrame = av_frame_alloc();
if (!scaledFrame) {
cerr << "Cannot allocate an AVPacket for encoded video" << endl;
exit(1);
}
outPacket = (AVPacket*)av_malloc(sizeof(AVPacket));
if (!outPacket) {
cerr << "Cannot allocate an AVPacket for encoded video" << endl;
exit(1);
}
//init the resampler
SwrContext* resampleContext = nullptr;
resampleContext = swr_alloc_set_opts(resampleContext,
av_get_default_channel_layout(outAudioCodecContext->channels),
outAudioCodecContext->sample_fmt,
outAudioCodecContext->sample_rate,
av_get_default_channel_layout(inAudioCodecContext->channels),
inAudioCodecContext->sample_fmt,
inAudioCodecContext->sample_rate,
0,
nullptr);
if (!resampleContext) {
cerr << "Cannot allocate the resample context" << endl;
exit(1);
}
if ((swr_init(resampleContext)) < 0) {
fprintf(stderr, "Could not open resample context\n");
swr_free(&resampleContext);
exit(1);
}
while (true) {
if (pauseCapture) {
cout << "Pause audio" << endl;
}
cv.wait(ul, [this]() { return !pauseCapture; });
if (stopCapture) {
break;
}
ul.unlock();
if (av_read_frame(inAudioFormatContext, inPacket) >= 0 && inPacket->stream_index == audioStreamIndx) {
//decode audio routing
av_packet_rescale_ts(outPacket, inAudioFormatContext->streams[audioStreamIndx]->time_base, inAudioCodecContext->time_base);
if ((ret = avcodec_send_packet(inAudioCodecContext, inPacket)) < 0) {
cout << "Cannot decode current audio packet " << ret << endl;
continue;
}
while (ret >= 0) {
ret = avcodec_receive_frame(inAudioCodecContext, rawFrame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
else if (ret < 0) {
cerr << "Error during decoding" << endl;
exit(1);
}
if (outAVFormatContext->streams[outAudioStreamIndex]->start_time <= 0) {
outAVFormatContext->streams[outAudioStreamIndex]->start_time = rawFrame->pts;
}
initConvertedSamples(&resampledData, outAudioCodecContext, rawFrame->nb_samples);
swr_convert(resampleContext,
resampledData, rawFrame->nb_samples,
(const uint8_t**)rawFrame->extended_data, rawFrame->nb_samp
add_samples_to_fifo(resampledData, rawFrame->nb_samples);
//raw frame ready
av_init_packet(outPacket);
outPacket->data = nullptr;
outPacket->size = 0;
const int frame_size = FFMAX(av_audio_fifo_size(fifo), outAudioCodecContext->frame_size);
scaledFrame = av_frame_alloc();
if (!scaledFrame) {
cerr << "Cannot allocate an AVPacket for encoded video" << endl;
exit(1);
}
scaledFrame->nb_samples = outAudioCodecContext->frame_size;
scaledFrame->channel_layout = outAudioCodecContext->channel_layout;
scaledFrame->format = outAudioCodecContext->sample_fmt;
scaledFrame->sample_rate = outAudioCodecContext->sample_rate;
av_frame_get_buffer(scaledFrame, 0);
while (av_audio_fifo_size(fifo) >= outAudioCodecContext->frame_size) {
ret = av_audio_fifo_read(fifo, (void**)(scaledFrame->data), outAudioCodecContext->frame_size);
scaledFrame->pts = pts;
pts += scaledFrame->nb_samples;
if (avcodec_send_frame(outAudioCodecContext, scaledFrame) < 0) {
cout << "Cannot encode current audio packet " << endl;
exit(1);
}
while (ret >= 0) {
ret = avcodec_receive_packet(outAudioCodecContext, outPacket);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
else if (ret < 0) {
cerr << "Error during encoding" << endl;
exit(1);
}
av_packet_rescale_ts(outPacket, outAudioCodecContext->time_base, outAVFormatContext->streams[outAudioStreamIndex]->time_base);
outPacket->stream_index = outAudioStreamIndex;
write_lock.lock();
if (av_write_frame(outAVFormatContext, outPacket) != 0)
{
cerr << "Error in writing audio frame" << endl;
}
write_lock.unlock();
av_packet_unref(outPacket);
}
ret = 0;
}
av_frame_free(&scaledFrame);
av_packet_unref(outPacket);
}
}
}
}
int ScreenRecorder::captureVideoFrames() {
int64_t pts = 0;
int flag;
int frameFinished = 0;
bool endPause = false;
int numPause = 0;
ofstream outFile{ "..\\media\\log.txt", ios::out };
int frameIndex = 0;
value = 0;
pAVPacket = (AVPacket*)av_malloc(sizeof(AVPacket));
if (pAVPacket == nullptr) {
cerr << "Error in allocating AVPacket" << endl;
exit(-1);
}
pAVFrame = av_frame_alloc();
if (pAVFrame == nullptr) {
cerr << "Error: unable to alloc the AVFrame resources" << endl;
exit(-1);
}
outFrame = av_frame_alloc();
if (outFrame == nullptr) {
cerr << "Error: unable to alloc the AVFrame resources for out frame" << endl;
exit(-1);
}
int videoOutBuffSize;
int nBytes = av_image_get_buffer_size(outVideoCodecContext->pix_fmt, outVideoCodecContext->width, outVideoCodecContext->height, 32);
uint8_t* videoOutBuff = (uint8_t*)av_malloc(nBytes);
if (videoOutBuff == nullptr) {
cerr << "Error: unable to allocate memory" << endl;
exit(-1);
}
value = av_image_fill_arrays(outFrame->data, outFrame->linesize, videoOutBuff, AV_PIX_FMT_YUV420P, outVideoCodecContext->width, outVideoCodecContext->height, 1);
if (value < 0) {
cerr << "Error in filling image array" << endl;
}
SwsContext* swsCtx_;
if (avcodec_open2(pAVCodecContext, pAVCodec, nullptr) < 0) {
cerr << "Could not open codec" << endl;
exit(-1);
}
swsCtx_ = sws_getContext(pAVCodecContext->width, pAVCodecContext->height, pAVCodecContext->pix_fmt, outVideoCodecContext->width, outVideoCodecContext->height, outVideoCodecContext->pix_fmt, SWS_BICUBIC,
nullptr, nullptr, nullptr);
AVPacket outPacket;
int gotPicture;
time_t startTime;
time(&startTime);
while (true) {
if (pauseCapture) {
cout << "Pause" << endl;
outFile << "/////////////////// Pause ///////////////////" << endl;
cout << "outVideoCodecContext->time_base: " << outVideoCodecContext->time_base.num << ", " << outVideoCodecContext->time_base.den << endl;
}
cv.wait(ul, [this]() { return !pauseCapture; }); //pause capture (not busy waiting)
if (endPause) {
endPause = false;
}
if (stopCapture) //check if the capture has to stop
break;
ul.unlock();
if (av_read_frame(pAVFormatContext, pAVPacket) >= 0 && pAVPacket->stream_index == VideoStreamIndx) {
av_packet_rescale_ts(pAVPacket, pAVFormatContext->streams[VideoStreamIndx]->time_base, pAVCodecContext->time_base);
value = avcodec_decode_video2(pAVCodecContext, pAVFrame, &frameFinished, pAVPacket);
if (value < 0) {
cout << "Unable to decode video" << endl;
}
if (frameFinished) { //frame successfully decoded
//sws_scale(swsCtx_, pAVFrame->data, pAVFrame->linesize, 0, pAVCodecContext->height, outFrame->data, outFrame->linesize);
av_init_packet(&outPacket);
outPacket.data = nullptr;
outPacket.size = 0;
if (outAVFormatContext->streams[outVideoStreamIndex]->start_time <= 0) {
outAVFormatContext->streams[outVideoStreamIndex]->start_time = pAVFrame->pts;
}
//disable warning on the console
outFrame->width = outVideoCodecContext->width;
outFrame->height = outVideoCodecContext->height;
outFrame->format = outVideoCodecContext->pix_fmt;
sws_scale(swsCtx_, pAVFrame->data, pAVFrame->linesize, 0, pAVCodecContext->height, outFrame->data, outFrame->linesize);
avcodec_encode_video2(outVideoCodecContext, &outPacket, outFrame, &gotPicture);
if (gotPicture) {
if (outPacket.pts != AV_NOPTS_VALUE) {
outPacket.pts = av_rescale_q(outPacket.pts, videoSt->codec->time_base, videoSt->time_base);
}
if (outPacket.dts != AV_NOPTS_VALUE) {
outPacket.dts = av_rescale_q(outPacket.dts, videoSt->codec->time_base, videoSt->time_base);
}
//cout << "Write frame " << j++ << " (size = " << outPacket.size / 1000 << ")" << endl;
//cout << "(size = " << outPacket.size << ")" << endl;
//av_packet_rescale_ts(&outPacket, outVideoCodecContext->time_base, outAVFormatContext->streams[outVideoStreamIndex]->time_base);
//outPacket.stream_index = outVideoStreamIndex;
outFile << "outPacket->duration: " << outPacket.duration << ", " << "pAVPacket->duration: " << pAVPacket->duration << endl;
outFile << "outPacket->pts: " << outPacket.pts << ", " << "pAVPacket->pts: " << pAVPacket->pts << endl;
outFile << "outPacket.dts: " << outPacket.dts << ", " << "pAVPacket->dts: " << pAVPacket->dts << endl;
time_t timer;
double seconds;
mu.lock();
if (!activeMenu) {
time(&timer);
seconds = difftime(timer, startTime);
int h = (int)(seconds / 3600);
int m = (int)(seconds / 60) % 60;
int s = (int)(seconds) % 60;
std::cout << std::flush << "\r" << std::setw(2) << std::setfill('0') << h << ':'
<< std::setw(2) << std::setfill('0') << m << ':'
<< std::setw(2) << std::setfill('0') << s << std::flush;
}
mu.unlock();
write_lock.lock();
if (av_write_frame(outAVFormatContext, &outPacket) != 0) {
cerr << "Error in writing video frame" << endl;
}
write_lock.unlock();
av_packet_unref(&outPacket);
}
av_packet_unref(&outPacket);
av_free_packet(pAVPacket); //avoid memory saturation
}
}
}
outFile.close();
av_free(videoOutBuff);
return 0;
}
I resolved this problem performing an avformat_close_input(&inAudioFormatContext) before enter in pause, and an avformat_open_input(&inAudioFormatContext, "audio=Microfono (Realtek(R) Audio)", audioInputFormat, &audioOptions) after resume the recording. In this way the final file seems well syncronized with video.
I try to set up parallel calculations in processes instead of threads. I started with a very simple example: send an integer from parent process to child process. It works and this is the code of the function processParallel1():
//parent sends integer to child
bool processParallel1()
{
//define pipe
int parent2Child[2];
//create pipe
pipe(parent2Child);
//fork
pid_t child = fork();
if(child == 0) //child process
{
//close not needed end of pipe
close(parent2Child[1]);
for(;;)
{
struct pollfd pfd;
pfd.fd = parent2Child[0];
pfd.events = POLLIN;
//wait until data is available at the pipe
if(poll(&pfd, 1, -1) < 0)
{
cout << strerror(errno) << endl;
return false;
}
if((pfd.revents&POLLIN) == POLLIN)
{
int data;
if(!readData1(parent2Child[0], data))
return false;
cout << "c: received data: " << strerror(errno) << endl;
cout << "data received: " << data << endl;
if(data == -1)
break;
}
}
exit(0);
}
else //parent process
{
//close not needed end of pipe
close(parent2Child[0]);
//send data to child
if(!writeData1(parent2Child[1], 136758))
return false;
cout << "p: wrote data: " << strerror(errno) << endl;
//send stop data
if(!writeData1(parent2Child[1], -1))
return false;
cout << "p: wrote data: " << strerror(errno) << endl;
//wait for child to end
wait(nullptr);
//close all pipes
close(parent2Child[0]);
close(parent2Child[1]);
}
return true;
}
In processParallel1() I use this methods for writing and reading to the pipe:
bool writeData1(int fd, const int data)
{
int bytesWritten = write(fd, &data, sizeof(int));
if(bytesWritten < 0)
{
cout << strerror(errno) << endl;
return false;
}
return true;
}
bool readData1(int fd, int & data)
{
char intBuf[sizeof(int)];
int bytesRead = read(fd, intBuf, sizeof(int));
if(bytesRead > 0)
{
data = *(int *)intBuf;
}
else if(bytesRead < 0)
{
cout << strerror(errno) << endl;
return false;
}
return true;
}
Everything works as expected. Now I add more data and it becomes strange. I only exchanged the methods for writing and reading (and created a new version of the method now named processParallel3()):
bool writeData3(int fd, const int number, const std::string text)
{
int bytesWritten = write(fd, &number, sizeof(int));
if(bytesWritten < 0)
{
cout << strerror(errno) << endl;
return false;
}
int size = text.size() + 1;
bytesWritten = write(fd, &size, sizeof(int));
if(bytesWritten < 0)
{
cout << strerror(errno) << endl;
return false;
}
bytesWritten = write(fd, text.c_str(), size);
if(bytesWritten < 0)
{
cout << strerror(errno) << endl;
return false;
}
return true;
}
bool readData3(int fd, int & number, std::string & text)
{
char numberBuf[sizeof(int)];
int bytesRead = read(fd, numberBuf, sizeof(int));
if(bytesRead > 0)
{
number = *(int *)numberBuf;
}
else if(bytesRead < 0)
{
cout << strerror(errno) << endl;
return false;
}
char sizeBuf[sizeof(int)];
int size = -1;
bytesRead = read(fd, sizeBuf, sizeof(int));
if(bytesRead > 0)
{
size = *(int *)sizeBuf;
}
else if(bytesRead < 0)
{
cout << strerror(errno) << endl;
return false;
}
char textBuf[size];
bytesRead = read(fd, textBuf, size);
if(bytesRead > 0)
{
text = std::string(textBuf);
}
else if(bytesRead < 0)
{
cout << strerror(errno) << endl;
return false;
}
return true;
}
When I run processParallel3() in a main I get three times a "Success" after writing and reading, but the methods hangs after printing the received data in the child.
When I add the method processParallel1() to be run directly before processParallel3() in 1 everything is ok, but now I get three times "Bad file descriptor" and it hangs again.
Any idea what could happen or what further tests I could run?
I am trying to read and most important to write metadata into file using ffmpeg. But I am getting sigseg.
int main(int argc, char **argv) {
av_register_all();
AVFormatContext* ctx;
std::string path("/home/stefan/test_track.mp3");
ctx = avformat_alloc_context();
if (avformat_open_input(&ctx, path.c_str(), 0, 0) < 0)
std::cout << "error1" << std::endl;
if (avformat_find_stream_info(ctx, 0) < 0)
std::cout << "error2" << std::endl;
AVDictionaryEntry *tag = nullptr;
tag = av_dict_get(ctx->metadata, "artist", tag, AV_DICT_IGNORE_SUFFIX);
std::cout << tag->key << " : " << tag->value << std::endl;
av_dict_set(&ctx->metadata, "TPFL", "testtest", 0);
std::cout << "test!" << std::endl;
int status = avformat_write_header(ctx, &ctx->metadata);
if(status == 0)
std::cout << "test1" << std::endl;
return 0;
}
I also tried to make a entire copy of AVDictionary and than saving it with new addition of field I want to save. But still sigseg.
What am I missing?
You need to use another AVFormatContext to write your output.
In your example, just add some meta data and copy the codec, so the muxing steps for ffmpeg library is
create the desired output format context, avformat_alloc_output_context2
add streams to the output format context, avformat_new_stream
add some custom meta data and write header
use av_write_frame to write the encoded data
write trailer
Below is a workable example with small modification of your code.
#include <iostream>
#ifdef __cplusplus
extern "C" {
#endif
#include <libavformat/avformat.h>
#ifdef __cplusplus
}
#endif
int main(int argc, char **argv) {
av_register_all();
avcodec_register_all();
AVFormatContext* ctx;
std::string path("./input.mp3");
ctx = avformat_alloc_context();
if (avformat_open_input(&ctx, path.c_str(), 0, 0) < 0)
std::cout << "error1" << std::endl;
if (avformat_find_stream_info(ctx, 0) < 0)
std::cout << "error2" << std::endl;
AVDictionaryEntry *tag = nullptr;
tag = av_dict_get(ctx->metadata, "artist", tag, AV_DICT_IGNORE_SUFFIX);
std::cout << tag->key << " : " << tag->value << std::endl;
av_dict_set(&ctx->metadata, "TPFL", "testtest", 0);
std::cout << "test!" << std::endl;
int status;
AVFormatContext* ofmt_ctx;
AVOutputFormat* ofmt = av_guess_format("mp3", "./out.mp3", NULL);
status = avformat_alloc_output_context2(&ofmt_ctx, ofmt, "mp3", "./out.mp3");
if (status < 0) {
std::cerr << "could not allocate output format" << std::endl;
return 0;
}
int audio_stream_index = 0;
for (unsigned i = 0; i < ctx->nb_streams; i++) {
if (ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
audio_stream_index = i;
const AVCodec *c = avcodec_find_encoder(ctx->streams[i]->codecpar->codec_id);
if (c) {
AVStream *ostream = avformat_new_stream(ofmt_ctx, c);
avcodec_parameters_copy(ostream->codecpar, ctx->streams[i]->codecpar);
ostream->codecpar->codec_tag = 0;
}
break;
}
}
av_dict_set(&ofmt_ctx->metadata, "TPFL", "testtest", 0);
av_dump_format(ofmt_ctx, 0, "./out.mp3", 1);
if (!(ofmt_ctx->oformat->flags & AVFMT_NOFILE)) {
avio_open(&ofmt_ctx->pb, "./out.mp3", AVIO_FLAG_WRITE);
}
if (avformat_init_output(ofmt_ctx, NULL) == AVSTREAM_INIT_IN_WRITE_HEADER) {
status = avformat_write_header(ofmt_ctx, NULL);
}
AVPacket *pkt = av_packet_alloc();
av_init_packet(pkt);
pkt->data = NULL;
pkt->size = 0;
while (av_read_frame(ctx, pkt) == 0) {
if (pkt->stream_index == audio_stream_index) {
// this is optional, we are copying the stream
av_packet_rescale_ts(pkt, ctx->streams[audio_stream_index]->time_base,
ofmt_ctx->streams[audio_stream_index]->time_base);
av_write_frame(ofmt_ctx, pkt);
}
}
av_packet_free(&pkt);
av_write_trailer(ofmt_ctx);
avformat_close_input(&ctx);
avformat_free_context(ofmt_ctx);
avformat_free_context(ctx);
if(status == 0)
std::cout << "test1" << std::endl;
return 0;
}
After done, you can use ffprobe output.mp3 to check the written meta data.
Input #0, mp3, from 'out.mp3':
Metadata:
TPFL : testtest
encoder : Lavf57.75.100
Duration: 00:00:08.75, start: 0.011995, bitrate: 128 kb/s
Stream #0:0: Audio: mp3, 44100 Hz, stereo, s16p, 128 kb/s
Metadata:
encoder : Lavf
I want to read binary file to OpenCL Memory Object using Zero Copy method. But, when my code execute readFile, my program has stopped working.
I have read other Q&A,
segmentation fault(core dumped) in opencl
How to avoid constant memory copying in OpenCL
But, I haven't solved my problem yet.
This is my code
//OPENCL Include
#include <CL/cl.h>
typedef float2 cplx;
int readFile(char *filen, cplx* data);
int main(int argc, char* argv[])
{
char *filename = (char*)malloc(100);
sprintf(filename,"%s",argv[1]);
//OpenCL Structures
cl_device_id device;
cl_context context;
cl_command_queue commandQueue;
int err;
int filesize = 1024;
device = create_device();
context = clCreateContext(NULL, 1, &device, NULL, NULL, &err);
if(err < 0)
{
perror("Couldn't create a context");
exit(1);
}
commandQueue = clCreateCommandQueue(context, device, 0, &err);
if(err < 0)
{
perror("Couldn't create Command Queue");
exit(1);
}
cl_mem memObj_data[1] = {0};
size_t buffer_size = 1024 * sizeof(cplx);
printf("Create Buffer\n");
memObj_data[0] = clCreateBuffer(context, CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR, buffer_size, NULL, &err);
if(err != CL_SUCCESS)
{
cerr << getErrorString(err) << endl;
exit(1);
}
// cplx *data = (cplx*)malloc(sizeof(cplx)*sizeFile);
// data = (cplx*)clEnqueueMapBuffer(commandQueue, memObj_data[0], CL_TRUE, CL_MAP_WRITE, 0, buffer_size, 0, NULL, NULL, &err);
printf("Enqueue Map Buffer\n");
cplx* data = (cplx*)clEnqueueMapBuffer(commandQueue, memObj_data[0], CL_TRUE, CL_MAP_WRITE, 0, buffer_size, 0, NULL, NULL, &err);
if(err != CL_SUCCESS)
{
cerr << getErrorString(err) << endl;
exit(1);
}
int ret = readFile(filename, data);
if(ret == 1)
{
cout << "Error on Reading File" << endl;
return 1;
printf("Enequeue Unmap Memory Object\n");
err = clEnqueueUnmapMemObject(commandQueue, memObj_data[0], data, 0, NULL, NULL);
if(err != CL_SUCCESS)
{
cerr << getErrorString(err) << endl;
exit(1);
}
//Deallocate resource
clReleaseMemObject(memObj_data[0]);
clReleaseCommandQueue(commandQueue);
clReleaseContext(context);
clReleaseDevice(device);
//FREE MEMORY
delete[] filename;
return 0;
}
int readFile(char *filen, cplx* data)
{
cout << "Read File" << endl;
//Get size contains of file
// ifstream file("../testfile", ios::in | ios::binary | ios::ate);
streampos size;
char path[20];
sprintf(path,"%s",filen);
ifstream file(path, ios::in | ios::binary | ios::ate);
if(file.is_open())
{
size = file.tellg();
// *data = (cplx*)realloc(*data, sizeof(cplx)*size);
if(size)
{
// data = data;
cout << "Read CFL file Success" << endl;
}
else
{
cout << "Error Allocating Memory" << endl;
return 1;
}
cout << "Contains Size : "<< std::to_string(size) << endl;
}
else
{
cout << "Unable to open file";
return 1;
}
if(file.is_open())
{
file.seekg(0, ios::beg);
file.read((char*)data, size);
file.close();
}
// int start = 230;
// int finish = 250;
// cout << "ON Functions" << endl;
// for(int i = start; i < finish; i++)
// {
// cout << i << " "<< std::to_string((*data)[i].x) << " + " << std::to_string((*data)[i].y) << endl;
// }
// data = memblock;
// free(memblock);
return 0;
}
Then, while read binary file, my program has stopped working.
Hi im trying to take sound from an open PortAudio Stream, encode it with opus, decode it and reproduce it again with portaudio.
Im doing this as a prototype just to try and understand the mechanics of this systems so, no real interest on following this concrete flow.
Thing is, portaudio gives buffers where OPUS needs Frames. Mi thought lead me to this in the portaudio side:
err = (Pa_ReadStream(stream, readBuffer, FRAMES_PER_BUFFER));
if (err = paNoError){
qDebug()<<"Fail read";
qDebug()<<Pa_GetErrorText(err);
// blockingRecord = false;
}
while (pos<FRAMES_PER_BUFFER){
memcpy(frameBuffer,readBuffer+(pos*FRAME_SIZE*NUM_CHANNELS),FRAME_SIZE*CHANNELS);
compressedSound = om.encodeOpus(frameBuffer);
unCompressedSound = om.decodeOpus(compressedSound);
memcpy(readBuffer+(pos*FRAME_SIZE*NUM_CHANNELS),unCompressedSound,FRAME_SIZE*CHANNELS);
pos++;
}
pos = 0;
err = (Pa_WriteStream(stream, readBuffer, FRAMES_PER_BUFFER));
if (err != paNoError)
{
qDebug() << "FAIL WRITE";
qDebug()<<Pa_GetErrorText(err);
//blockingRecord = false;
}
And this on the OPUS side:
unsigned char * OpusManager::encodeOpus(unsigned char *frame){
memcpy(encoded, frame, FRAME_SIZE*CHANNELS);
int ret = opus_encode(enc, encoded, FRAME_SIZE, compressed_buffer, encoded_data_size);
if (ret<0){
qDebug()<<"Failure while compressing sound";
return NULL;
}
return (compressed_buffer);
}
unsigned char * OpusManager::decodeOpus(unsigned char *frame){
int ret= opus_decode(dec, frame, encoded_data_size, decoded, FRAME_SIZE, 0);
if (ret<0){
qDebug()<<"Failure while decompressing sound";
return NULL;
}
memcpy(uncompressed_buffer, decoded, FRAME_SIZE*CHANNELS);
return (uncompressed_buffer);
}
No errors without encocing and perfect soud. With encode i get no errors till the PA_Writestream call, where i get a "Output underflowed" PaError. I suppose the way of taking the frames ive implemmented must be waaay wrong, but cant find info to help me with this.
It seems your interpretation of Opus' frame_size parameters to opus_encode and opus_decode is incorrect. If I understand your code correctly you're recording a packet of size FRAMES_PER_BUFFER frames and then try to turn it into N packets of size FRAME_SIZE. Instead, it seems to me that Opus wants to turn your packet of FRAMES_PER_BUFFER into another packet of equal frame count, and in doing so, only uses it's FRAME_SIZE parameter as some sort of quality control parameter for the encoding process. Below you'll find a complete sample that I believe does what you want. Play around with the '480' magic number in encode()/decode() and hear audio quality change.
int opusErr;
PaError paErr;
std::string s;
int const channels = 2;
int const bufferSize = 480;
int const sampleRate = 48000;
int const durationSeconds = 5;
opus_int32 enc_bytes;
opus_int32 dec_bytes;
int framesProcessed = 0;
std::vector<unsigned short> captured(bufferSize * channels);
std::vector<unsigned short> decoded(bufferSize * channels);
// * 2: byte count, 16 bit samples
std::vector<unsigned char> encoded(bufferSize * channels * 2);
// initialize opus
OpusEncoder* enc = opus_encoder_create(
sampleRate, channels, OPUS_APPLICATION_AUDIO, &opusErr);
if (opusErr != OPUS_OK)
{
std::cout << "opus_encoder_create failed: " << opusErr << "\n";
std::getline(std::cin, s);
return 1;
}
OpusDecoder* dec = opus_decoder_create(
sampleRate, channels, &opusErr);
if (opusErr != OPUS_OK)
{
std::cout << "opus_decoder_create failed: " << opusErr << "\n";
std::getline(std::cin, s);
return 1;
}
// initialize portaudio
if ((paErr = Pa_Initialize()) != paNoError)
{
std::cout << "Pa_Initialize failed: " << Pa_GetErrorText(paErr) << "\n";
std::getline(std::cin, s);
return 1;
}
PaStream* stream = nullptr;
if ((paErr = Pa_OpenDefaultStream(&stream,
channels, channels, paInt16, sampleRate,
bufferSize, nullptr, nullptr)) != paNoError)
{
std::cout << "Pa_OpenDefaultStream failed: " << Pa_GetErrorText(paErr) << "\n";
std::getline(std::cin, s);
return 1;
}
// start stream
if ((paErr = Pa_StartStream(stream)) != paNoError)
{
std::cout << "Pa_StartStream failed: " << Pa_GetErrorText(paErr) << "\n";
std::getline(std::cin, s);
return 1;
}
// capture, encode, decode & render durationSeconds of audio
while (framesProcessed < sampleRate * durationSeconds)
{
if ((paErr = Pa_ReadStream(stream,
captured.data(), bufferSize)) != paNoError)
{
std::cout << "Pa_ReadStream failed: " << Pa_GetErrorText(paErr) << "\n";
std::getline(std::cin, s);
return 1;
}
if ((enc_bytes = opus_encode(enc, reinterpret_cast<opus_int16 const*>(
captured.data()), 480, encoded.data(), encoded.size())) < 0)
{
std::cout << "opus_encode failed: " << enc_bytes << "\n";
std::getline(std::cin, s);
return 1;
}
if ((dec_bytes = opus_decode(dec, encoded.data(), enc_bytes,
reinterpret_cast<opus_int16*>(decoded.data()), 480, 0)) < 0)
{
std::cout << "opus_decode failed: " << dec_bytes << "\n";
std::getline(std::cin, s);
return 1;
}
if ((paErr = Pa_WriteStream(stream, decoded.data(), bufferSize)) != paNoError)
{
std::cout << "Pa_WriteStream failed: " << Pa_GetErrorText(paErr) << "\n";
std::getline(std::cin, s);
return 1;
}
framesProcessed += bufferSize;
}
// stop stream
if ((paErr = Pa_StopStream(stream)) != paNoError)
{
std::cout << "Pa_StopStream failed: " << Pa_GetErrorText(paErr) << "\n";
std::getline(std::cin, s);
return 1;
}
// cleanup portaudio
if ((paErr = Pa_CloseStream(stream)) != paNoError)
{
std::cout << "Pa_CloseStream failed: " << Pa_GetErrorText(paErr) << "\n";
std::getline(std::cin, s);
return 1;
}
if ((paErr = Pa_Terminate()) != paNoError)
{
std::cout << "Pa_Terminate failed: " << Pa_GetErrorText(paErr) << "\n";
std::getline(std::cin, s);
return 1;
}
// cleanup opus
opus_decoder_destroy(dec);
opus_encoder_destroy(enc);