I am compiling the openAL tools on an OS X system and having troubles building the example files.
I am getting the error:
Undefined symbols for architecture x86_64:
"_main", referenced from:
implicit entry/start for main executable
(maybe you meant: _SDL_main)
ld: symbol(s) not found for architecture x86_64
When compiling on an linux machine, I'm getting no errors and the example binaries just work fine. What am I doing wrong?
That is the code of the program I am compiling:
#include <stdio.h>
#include <assert.h>
#include <math.h>
#include "AL/al.h"
#include "AL/alc.h"
#include "AL/alext.h"
#include "common/alhelpers.h"
#include "common/sdl_sound.h"
#ifndef M_PI
#define M_PI (3.14159265358979323846)
#endif
static LPALCGETSTRINGISOFT alcGetStringiSOFT;
static LPALCRESETDEVICESOFT alcResetDeviceSOFT;
/* LoadBuffer loads the named audio file into an OpenAL buffer object, and
* returns the new buffer ID. */
static ALuint LoadSound(const char *filename)
{
ALenum err, format, type, channels;
ALuint rate, buffer;
size_t datalen;
void *data;
FilePtr sound;
/* Open the audio file */
sound = openAudioFile(filename, 1000);
if(!sound)
{
fprintf(stderr, "Could not open audio in %s\n", filename);
closeAudioFile(sound);
return 0;
}
/* Get the sound format, and figure out the OpenAL format */
if(getAudioInfo(sound, &rate, &channels, &type) != 0)
{
fprintf(stderr, "Error getting audio info for %s\n", filename);
closeAudioFile(sound);
return 0;
}
format = GetFormat(channels, type, NULL);
if(format == AL_NONE)
{
fprintf(stderr, "Unsupported format (%s, %s) for %s\n",
ChannelsName(channels), TypeName(type), filename);
closeAudioFile(sound);
return 0;
}
/* Decode the whole audio stream to a buffer. */
data = decodeAudioStream(sound, &datalen);
if(!data)
{
fprintf(stderr, "Failed to read audio from %s\n", filename);
closeAudioFile(sound);
return 0;
}
/* Buffer the audio data into a new buffer object, then free the data and
* close the file. */
buffer = 0;
alGenBuffers(1, &buffer);
alBufferData(buffer, format, data, datalen, rate);
free(data);
closeAudioFile(sound);
/* Check if an error occured, and clean up if so. */
err = alGetError();
if(err != AL_NO_ERROR)
{
fprintf(stderr, "OpenAL Error: %s\n", alGetString(err));
if(buffer && alIsBuffer(buffer))
alDeleteBuffers(1, &buffer);
return 0;
}
return buffer;
}
int main(int argc, char **argv)
{
ALCdevice *device;
ALuint source, buffer;
const char *soundname;
const char *hrtfname;
ALCint hrtf_state;
ALCint num_hrtf;
ALdouble angle;
ALenum state;
/* Print out usage if no file was specified */
if(argc < 2 || (strcmp(argv[1], "-hrtf") == 0 && argc < 4))
{
fprintf(stderr, "Usage: %s [-hrtf <name>] <soundfile>\n", argv[0]);
return 1;
}
/* Initialize OpenAL with the default device, and check for HRTF support. */
if(InitAL() != 0)
return 1;
if(strcmp(argv[1], "-hrtf") == 0)
{
hrtfname = argv[2];
soundname = argv[3];
}
else
{
hrtfname = NULL;
soundname = argv[1];
}
device = alcGetContextsDevice(alcGetCurrentContext());
if(!alcIsExtensionPresent(device, "ALC_SOFT_HRTF"))
{
fprintf(stderr, "Error: ALC_SOFT_HRTF not supported\n");
CloseAL();
return 1;
}
/* Define a macro to help load the function pointers. */
#define LOAD_PROC(d, x) ((x) = alcGetProcAddress((d), #x))
LOAD_PROC(device, alcGetStringiSOFT);
LOAD_PROC(device, alcResetDeviceSOFT);
#undef LOAD_PROC
/* Enumerate available HRTFs, and reset the device using one. */
alcGetIntegerv(device, ALC_NUM_HRTF_SPECIFIERS_SOFT, 1, &num_hrtf);
if(!num_hrtf)
printf("No HRTFs found\n");
else
{
ALCint attr[5];
ALCint index = -1;
ALCint i;
printf("Available HRTFs:\n");
for(i = 0;i < num_hrtf;i++)
{
const ALCchar *name = alcGetStringiSOFT(device, ALC_HRTF_SPECIFIER_SOFT, i);
printf(" %d: %s\n", i, name);
/* Check if this is the HRTF the user requested. */
if(hrtfname && strcmp(name, hrtfname) == 0)
index = i;
}
i = 0;
attr[i++] = ALC_HRTF_SOFT;
attr[i++] = ALC_TRUE;
if(index == -1)
{
if(hrtfname)
printf("HRTF \"%s\" not found\n", hrtfname);
printf("Using default HRTF...\n");
}
else
{
printf("Selecting HRTF %d...\n", index);
attr[i++] = ALC_HRTF_ID_SOFT;
attr[i++] = index;
}
attr[i] = 0;
if(!alcResetDeviceSOFT(device, attr))
printf("Failed to reset device: %s\n", alcGetString(device, alcGetError(device)));
}
/* Check if HRTF is enabled, and show which is being used. */
alcGetIntegerv(device, ALC_HRTF_SOFT, 1, &hrtf_state);
if(!hrtf_state)
printf("HRTF not enabled!\n");
else
{
const ALchar *name = alcGetString(device, ALC_HRTF_SPECIFIER_SOFT);
printf("HRTF enabled, using %s\n", name);
}
fflush(stdout);
/* Load the sound into a buffer. */
buffer = LoadSound(soundname);
if(!buffer)
{
CloseAL();
return 1;
}
/* Create the source to play the sound with. */
source = 0;
alGenSources(1, &source);
alSourcei(source, AL_SOURCE_RELATIVE, AL_TRUE);
alSource3f(source, AL_POSITION, 0.0f, 0.0f, -1.0f);
alSourcei(source, AL_BUFFER, buffer);
assert(alGetError()==AL_NO_ERROR && "Failed to setup sound source");
/* Play the sound until it finishes. */
angle = 0.0;
alSourcePlay(source);
do {
al_nssleep(10000000);
/* Rotate the source around the listener by about 1/4 cycle per second.
* Only affects mono sounds.
*/
angle += 0.01 * M_PI * 0.5;
alSource3f(source, AL_POSITION, (ALfloat)sin(angle), 0.0f, -(ALfloat)cos(angle));
alGetSourcei(source, AL_SOURCE_STATE, &state);
} while(alGetError() == AL_NO_ERROR && state == AL_PLAYING);
/* All done. Delete resources, and close OpenAL. */
alDeleteSources(1, &source);
alDeleteBuffers(1, &buffer);
CloseAL();
return 0;
}
And that is the linker command (automatically produced by cmake):
/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/cc -std=c11 -O2 -g -D_DEBUG -Wl,-search_paths_first -Wl,-headerpad_max_install_names CMakeFiles/alhrtf.dir/examples/alhrtf.c.o -o alhrtf libex-common.a /usr/local/lib/libSDL_sound.dylib -framework SDL2 -framework Cocoa -framework SDL2 -framework Cocoa libcommon.a libopenal.1.17.2.dylib libcommon.a -framework AudioToolbox -framework ApplicationServices -framework AudioUnit -framework CoreAudio -lpthread -ldl -lm -pthread -Wl,-rpath,/Library/Frameworks -Wl,-rpath,/Users/tschekar/Downloads/openal-soft/cmake
Related
One of the things I'm trying to achieve is parallel encoding via FFmpeg's c API. This looks to work out of the box quite nicely; however, I've changed the goal posts slightly:
In an existing application, I already have a thread pool at hand. Instead of using another thread pool via FFmpeg, I would like reuse the existing thread pool in my application. Having studied the latest FFmpeg trunk docs, it very much looks possible.
Using some FFmpeg sample code, I've created a sample application to demonstrate what I'm trying to achieve (see below). The sample app generates a video-only mpeg2 ts using the mp2v codec.
The problem I'm experiencing is that the custom 'thread_execute' or 'thread_execute2' are never invoked. This is despite the fact that the codec appears to indicate that threading is supported. Please be aware that I have not yet plumbed in the thread pool just yet. My first goal is for it to call the custom function pointer.
I've tried to get assistance on the FFmpeg mailing lists but to no avail.
#include <iostream>
#include <thread>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <cstring>
#include <future>
extern "C"
{
#include <libavutil/avassert.h>
#include <libavutil/channel_layout.h>
#include <libavutil/opt.h>
#include <libavutil/timestamp.h>
#include <libavformat/avformat.h>
//#include <libswscale/swscale.h>
#include <libswresample/swresample.h>
}
#define STREAM_DURATION 1000.0
#define STREAM_FRAME_RATE 25 /* 25 images/s */
#define STREAM_PIX_FMT AV_PIX_FMT_YUV420P /* default pix_fmt */
#define SCALE_FLAGS SWS_BICUBIC
// a wrapper around a single output AVStream
typedef struct OutputStream {
AVStream *st;
AVCodecContext *enc;
/* pts of the next frame that will be generated */
int64_t next_pts;
int samples_count;
AVFrame *frame;
AVFrame *tmp_frame;
float t, tincr, tincr2;
struct SwsContext *sws_ctx;
struct SwrContext *swr_ctx;
} OutputStream;
/////////////////////////////////////////////////////////////////////////////
// The ffmpeg variation raises compiler warnings.
char *cb_av_ts2str(char *buf, int64_t ts)
{
std::memset(buf,0,AV_TS_MAX_STRING_SIZE);
return av_ts_make_string(buf,ts);
}
/////////////////////////////////////////////////////////////////////////////
// The ffmpeg variation raises compiler warnings.
char *cb_av_ts2timestr(char *buf, int64_t ts, AVRational *tb)
{
std::memset(buf,0,sizeof(AV_TS_MAX_STRING_SIZE));
return av_ts_make_time_string(buf,ts,tb);
}
/////////////////////////////////////////////////////////////////////////////
// The ffmpeg variation raises compiler warnings.
char *cb_av_err2str(char *errbuf, size_t errbuf_size, int errnum)
{
std::memset(errbuf,0,errbuf_size);
return av_make_error_string(errbuf,errbuf_size,errnum);
}
int thread_execute(AVCodecContext* s, int (*func)(AVCodecContext *c2, void *arg2), void* arg, int* ret, int count, int size)
{
// Do it all serially for now
std::cout << "thread_execute" << std::endl;
for (int k = 0; k < count; ++k)
{
ret[k] = func(s, arg);
}
return 0;
}
int thread_execute2(AVCodecContext* s, int (*func)(AVCodecContext* c2, void* arg2, int, int), void* arg, int* ret, int count)
{
// Do it all serially for now
std::cout << "thread_execute2" << std::endl;
for (int k = 0; k < count; ++k)
{
ret[k] = func(s, arg, k, count);
}
return 0;
}
static void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt)
{
char s[AV_TS_MAX_STRING_SIZE];
AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
printf("pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
cb_av_ts2str(s,pkt->pts), cb_av_ts2timestr(s,pkt->pts, time_base),
cb_av_ts2str(s,pkt->dts), cb_av_ts2timestr(s,pkt->dts, time_base),
cb_av_ts2str(s,pkt->duration), cb_av_ts2timestr(s,pkt->duration, time_base),
pkt->stream_index);
}
static int write_frame(AVFormatContext *fmt_ctx, const AVRational *time_base, AVStream *st, AVPacket *pkt)
{
/* rescale output packet timestamp values from codec to stream timebase */
av_packet_rescale_ts(pkt, *time_base, st->time_base);
pkt->stream_index = st->index;
/* Write the compressed frame to the media file. */
log_packet(fmt_ctx, pkt);
return av_interleaved_write_frame(fmt_ctx, pkt);
}
/* Add an output stream. */
static void add_stream(OutputStream *ost, AVFormatContext *oc,
AVCodec **codec,
enum AVCodecID codec_id)
{
AVCodecContext *c;
int i;
/* find the encoder */
*codec = avcodec_find_encoder(codec_id);
if (!(*codec)) {
fprintf(stderr, "Could not find encoder for '%s'\n",
avcodec_get_name(codec_id));
exit(1);
}
ost->st = avformat_new_stream(oc, NULL);
if (!ost->st) {
fprintf(stderr, "Could not allocate stream\n");
exit(1);
}
ost->st->id = oc->nb_streams-1;
c = avcodec_alloc_context3(*codec);
if (!c) {
fprintf(stderr, "Could not alloc an encoding context\n");
exit(1);
}
ost->enc = c;
switch ((*codec)->type)
{
case AVMEDIA_TYPE_AUDIO:
c->sample_fmt = (*codec)->sample_fmts ?
(*codec)->sample_fmts[0] : AV_SAMPLE_FMT_FLTP;
c->bit_rate = 64000;
c->sample_rate = 44100;
if ((*codec)->supported_samplerates) {
c->sample_rate = (*codec)->supported_samplerates[0];
for (i = 0; (*codec)->supported_samplerates[i]; i++) {
if ((*codec)->supported_samplerates[i] == 44100)
c->sample_rate = 44100;
}
}
c->channels = av_get_channel_layout_nb_channels(c->channel_layout);
c->channel_layout = AV_CH_LAYOUT_STEREO;
if ((*codec)->channel_layouts) {
c->channel_layout = (*codec)->channel_layouts[0];
for (i = 0; (*codec)->channel_layouts[i]; i++) {
if ((*codec)->channel_layouts[i] == AV_CH_LAYOUT_STEREO)
c->channel_layout = AV_CH_LAYOUT_STEREO;
}
}
c->channels = av_get_channel_layout_nb_channels(c->channel_layout);
ost->st->time_base = (AVRational){ 1, c->sample_rate };
break;
case AVMEDIA_TYPE_VIDEO:
c->codec_id = codec_id;
c->bit_rate = 400000;
/* Resolution must be a multiple of two. */
c->width = 352;
c->height = 288;
/* timebase: This is the fundamental unit of time (in seconds) in terms
* of which frame timestamps are represented. For fixed-fps content,
* timebase should be 1/framerate and timestamp increments should be
* identical to 1. */
ost->st->time_base = (AVRational){ 1, STREAM_FRAME_RATE };
c->time_base = ost->st->time_base;
c->gop_size = 12; /* emit one intra frame every twelve frames at most */
c->pix_fmt = STREAM_PIX_FMT;
if (c->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
/* just for testing, we also add B-frames */
c->max_b_frames = 2;
}
if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) {
/* Needed to avoid using macroblocks in which some coeffs overflow.
* This does not happen with normal video, it just happens here as
* the motion of the chroma plane does not match the luma plane. */
c->mb_decision = 2;
}
break;
default:
break;
}
if (c->codec->capabilities & AV_CODEC_CAP_FRAME_THREADS ||
c->codec->capabilities & AV_CODEC_CAP_SLICE_THREADS)
{
if (c->codec->capabilities & AV_CODEC_CAP_FRAME_THREADS)
{
c->thread_type = FF_THREAD_FRAME;
}
if (c->codec->capabilities & AV_CODEC_CAP_SLICE_THREADS)
{
c->thread_type = FF_THREAD_SLICE;
}
c->execute = &thread_execute;
c->execute2 = &thread_execute2;
c->thread_count = 4;
// NOTE: Testing opaque.
c->opaque = (void*)0xff;
}
/* Some formats want stream headers to be separate. */
if (oc->oformat->flags & AVFMT_GLOBALHEADER)
c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
/**************************************************************/
/* video output */
static AVFrame *alloc_picture(enum AVPixelFormat pix_fmt, int width, int height)
{
AVFrame *picture;
int ret;
picture = av_frame_alloc();
if (!picture)
return NULL;
picture->format = pix_fmt;
picture->width = width;
picture->height = height;
/* allocate the buffers for the frame data */
ret = av_frame_get_buffer(picture, 32);
if (ret < 0) {
fprintf(stderr, "Could not allocate frame data.\n");
exit(1);
}
return picture;
}
static void open_video(AVFormatContext *oc, AVCodec *codec, OutputStream *ost, AVDictionary *opt_arg)
{
int ret;
AVCodecContext *c = ost->enc;
//AVDictionary *opt = NULL;
//av_dict_copy(&opt, opt_arg, 0);
/* open the codec */
ret = avcodec_open2(c, codec, NULL);
//av_dict_free(&opt);
if (ret < 0) {
char s[AV_ERROR_MAX_STRING_SIZE];
fprintf(stderr, "Could not open video codec: %s\n", cb_av_err2str(s,AV_ERROR_MAX_STRING_SIZE,ret));
exit(1);
}
/* allocate and init a re-usable frame */
ost->frame = alloc_picture(c->pix_fmt, c->width, c->height);
if (!ost->frame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
/* If the output format is not YUV420P, then a temporary YUV420P
* picture is needed too. It is then converted to the required
* output format. */
ost->tmp_frame = NULL;
if (c->pix_fmt != AV_PIX_FMT_YUV420P) {
ost->tmp_frame = alloc_picture(AV_PIX_FMT_YUV420P, c->width, c->height);
if (!ost->tmp_frame) {
fprintf(stderr, "Could not allocate temporary picture\n");
exit(1);
}
}
/* copy the stream parameters to the muxer */
ret = avcodec_parameters_from_context(ost->st->codecpar, c);
if (ret < 0) {
fprintf(stderr, "Could not copy the stream parameters\n");
exit(1);
}
}
/* Prepare a dummy image. */
static void fill_yuv_image(AVFrame *pict, int frame_index,
int width, int height)
{
int x, y, i;
i = frame_index;
/* Y */
for (y = 0; y < height; y++)
for (x = 0; x < width; x++)
pict->data[0][y * pict->linesize[0] + x] = x + y + i * 3;
/* Cb and Cr */
for (y = 0; y < height / 2; y++) {
for (x = 0; x < width / 2; x++) {
pict->data[1][y * pict->linesize[1] + x] = 128 + y + i * 2;
pict->data[2][y * pict->linesize[2] + x] = 64 + x + i * 5;
}
}
}
static AVFrame *get_video_frame(OutputStream *ost)
{
AVCodecContext *c = ost->enc;
/* check if we want to generate more frames */
if (av_compare_ts(ost->next_pts, c->time_base,
STREAM_DURATION, (AVRational){ 1, 1 }) >= 0)
return NULL;
/* when we pass a frame to the encoder, it may keep a reference to it
* internally; make sure we do not overwrite it here */
if (av_frame_make_writable(ost->frame) < 0)
exit(1);
if (c->pix_fmt != AV_PIX_FMT_YUV420P) {
/* as we only generate a YUV420P picture, we must convert it
* to the codec pixel format if needed */
/*if (!ost->sws_ctx) {
ost->sws_ctx = sws_getContext(c->width, c->height,
AV_PIX_FMT_YUV420P,
c->width, c->height,
c->pix_fmt,
SCALE_FLAGS, NULL, NULL, NULL);
if (!ost->sws_ctx) {
fprintf(stderr,
"Could not initialize the conversion context\n");
exit(1);
}
}
fill_yuv_image(ost->tmp_frame, ost->next_pts, c->width, c->height);
sws_scale(ost->sws_ctx,
(const uint8_t * const *)ost->tmp_frame->data, ost->tmp_frame->linesize,
0, c->height, ost->frame->data, ost->frame->linesize);*/
} else {
fill_yuv_image(ost->frame, ost->next_pts, c->width, c->height);
}
ost->frame->pts = ost->next_pts++;
return ost->frame;
}
/*
* encode one video frame and send it to the muxer
* return 1 when encoding is finished, 0 otherwise
*/
static int write_video_frame(AVFormatContext *oc, OutputStream *ost)
{
int ret;
AVCodecContext *c;
AVFrame *frame;
int got_packet = 0;
AVPacket pkt = { 0 };
c = ost->enc;
frame = get_video_frame(ost);
if (frame)
{
ret = avcodec_send_frame(ost->enc, frame);
if (ret < 0)
{
char s[AV_ERROR_MAX_STRING_SIZE];
fprintf(stderr, "Error encoding video frame: %s\n", cb_av_err2str(s, AV_ERROR_MAX_STRING_SIZE, ret));
exit(1);
}
}
av_init_packet(&pkt);
ret = avcodec_receive_packet(ost->enc,&pkt);
if (ret < 0)
{
if (ret == AVERROR(EAGAIN)) { ret = 0; }
else
{
char s[AV_ERROR_MAX_STRING_SIZE];
fprintf(stderr, "Error receiving packet: %s\n", cb_av_err2str(s,AV_ERROR_MAX_STRING_SIZE,ret));
exit(1);
}
}
else
{
got_packet = 1;
ret = write_frame(oc, &c->time_base, ost->st, &pkt);
}
if (ret < 0) {
char s[AV_ERROR_MAX_STRING_SIZE];
fprintf(stderr, "Error while writing video frame: %s\n", cb_av_err2str(s,AV_ERROR_MAX_STRING_SIZE,ret));
exit(1);
}
return (frame || got_packet) ? 0 : 1;
}
static void close_stream(AVFormatContext *oc, OutputStream *ost)
{
avcodec_free_context(&ost->enc);
av_frame_free(&ost->frame);
av_frame_free(&ost->tmp_frame);
//sws_freeContext(ost->sws_ctx);
//swr_free(&ost->swr_ctx);
}
/**************************************************************/
/* media file output */
int main(int argc, char **argv)
{
OutputStream video_st = { 0 }, audio_st = { 0 };
const char *filename;
AVOutputFormat *fmt;
AVFormatContext *oc;
AVCodec /**audio_codec,*/ *video_codec;
int ret;
int have_video = 0, have_audio = 0;
int encode_video = 0, encode_audio = 0;
AVDictionary *opt = NULL;
int i;
/* Initialize libavcodec, and register all codecs and formats. */
av_register_all();
avformat_network_init();
if (argc < 2) {
printf("usage: %s output_file\n"
"API example program to output a media file with libavformat.\n"
"This program generates a synthetic audio and video stream, encodes and\n"
"muxes them into a file named output_file.\n"
"The output format is automatically guessed according to the file extension.\n"
"Raw images can also be output by using '%%d' in the filename.\n"
"\n", argv[0]);
return 1;
}
filename = argv[1];
for (i = 2; i+1 < argc; i+=2) {
if (!strcmp(argv[i], "-flags") || !strcmp(argv[i], "-fflags"))
av_dict_set(&opt, argv[i]+1, argv[i+1], 0);
}
const char *pfilename = filename;
/* allocate the output media context */
avformat_alloc_output_context2(&oc, NULL, "mpegts", pfilename);
if (!oc) {
printf("Could not deduce output format from file extension: using MPEG.\n");
avformat_alloc_output_context2(&oc, NULL, "mpeg", pfilename);
}
if (!oc)
return 1;
fmt = oc->oformat;
/* Add the audio and video streams using the default format codecs
* and initialize the codecs. */
if (fmt->video_codec != AV_CODEC_ID_NONE) {
add_stream(&video_st, oc, &video_codec, fmt->video_codec);
have_video = 1;
encode_video = 1;
}
/*if (fmt->audio_codec != AV_CODEC_ID_NONE) {
add_stream(&audio_st, oc, &audio_codec, fmt->audio_codec);
have_audio = 1;
encode_audio = 1;
}*/
/* Now that all the parameters are set, we can open the audio and
* video codecs and allocate the necessary encode buffers. */
if (have_video)
open_video(oc, video_codec, &video_st, opt);
//if (have_audio)
// open_audio(oc, audio_codec, &audio_st, opt);
av_dump_format(oc, 0, pfilename, 1);
/* open the output file, if needed */
if (!(fmt->flags & AVFMT_NOFILE)) {
ret = avio_open(&oc->pb, pfilename, AVIO_FLAG_WRITE);
if (ret < 0) {
char s[AV_ERROR_MAX_STRING_SIZE];
fprintf(stderr, "Could not open '%s': %s\n", pfilename,
cb_av_err2str(s,AV_ERROR_MAX_STRING_SIZE,ret));
return 1;
}
}
/* Write the stream header, if any. */
ret = avformat_write_header(oc, &opt);
if (ret < 0) {
char s[AV_ERROR_MAX_STRING_SIZE];
fprintf(stderr, "Error occurred when opening output file: %s\n",
cb_av_err2str(s,AV_ERROR_MAX_STRING_SIZE,ret));
return 1;
}
while (encode_video || encode_audio) {
/* select the stream to encode */
if (encode_video &&
(!encode_audio || av_compare_ts(video_st.next_pts, video_st.enc->time_base,
audio_st.next_pts, audio_st.enc->time_base) <= 0)) {
encode_video = !write_video_frame(oc, &video_st);
} else {
//encode_audio = !write_audio_frame(oc, &audio_st);
}
//std::this_thread::sleep_for(std::chrono::milliseconds(35));
}
/* Write the trailer, if any. The trailer must be written before you
* close the CodecContexts open when you wrote the header; otherwise
* av_write_trailer() may try to use memory that was freed on
* av_codec_close(). */
av_write_trailer(oc);
/* Close each codec. */
if (have_video)
close_stream(oc, &video_st);
if (have_audio)
close_stream(oc, &audio_st);
if (!(fmt->flags & AVFMT_NOFILE))
/* Close the output file. */
avio_closep(&oc->pb);
/* free the stream */
avformat_free_context(oc);
return 0;
}
//
Environment:
Ubuntu Zesty (17.04)
FFmpeg version 3.2.4 (via package manager)
gcc 6.3 (C++)
You have to do following:
call avcodec_alloc_context3(...). This call will set default execute and execute2 functions in new context
set c->thread_count = number_of_threads_in_your_thread_pool()
call avcodec_open2(...).
set c->execute and c->execute2 to point to your functions
call ff_thread_free(c). This function isnt exposed in libavcodec headers but you can add following line:
extern "C" void ff_thread_free(AVCodecContext *s);
Drawback is that libavcodec will create internal thread pool after avcodec_open2(...) call, and that pool will be deleted in ff_thread_free() call.
Internal thread pool is very efficient, but its not good if you plan to do parallel encoding of multiple video feeds. In that case libavcodec will create separate thread pool for each encoding video feed.
I want to decode H.264 video from a collection of MPEG-2 Transport Stream packets but I am not clear what to pass to avcodec_decode_video2
The documentation says to pass "the input AVPacket containing the input buffer."
But what should be in the input buffer?
A PES packet will be spread across the payload portion of several TS packets, with NALU(s) inside the PES. So pass a TS fragment? The entire PES? PES payload only?
This Sample Code mentions:
BUT some other codecs (msmpeg4, mpeg4) are inherently frame based, so
you must call them with all the data for one frame exactly. You must
also initialize 'width' and 'height' before initializing them.
But I can find no info on what "all the data" means...
Passing a fragment of a TS packet payload is not working:
AVPacket avDecPkt;
av_init_packet(&avDecPkt);
avDecPkt.data = inbuf_ptr;
avDecPkt.size = esBufSize;
len = avcodec_decode_video2(mpDecoderContext, mpFrameDec, &got_picture, &avDecPkt);
if (len < 0)
{
printf(" TS PKT #%.0f. Error decoding frame #%04d [rc=%d '%s']\n",
tsPacket.pktNum, mDecodedFrameNum, len, av_make_error_string(errMsg, 128, len));
return;
}
output
[h264 # 0x81cd2a0] no frame!
TS PKT #2973. Error decoding frame #0001 [rc=-1094995529 'Invalid data found when processing input']
EDIT
Using the excellent hits from WLGfx, I made this simple program to try decoding TS packets. As input, I prepared a file containing only TS packets from the Video PID.
It feels close but I don't know how to set up the FormatContext. The code below segfaults at av_read_frame() (and internally at ret = s->iformat->read_packet(s, pkt)). s->iformat is zero.
Suggestions?
EDIT II - Sorry, for got post source code **
**EDIT III - Sample code updated to simulate reading TS PKT Queue
/*
* Test program for video decoder
*/
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
extern "C" {
#ifdef __cplusplus
#define __STDC_CONSTANT_MACROS
#ifdef _STDINT_H
#undef _STDINT_H
#endif
#include <stdint.h>
#endif
}
extern "C" {
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
#include "libavutil/imgutils.h"
#include "libavutil/opt.h"
}
class VideoDecoder
{
public:
VideoDecoder();
bool rcvTsPacket(AVPacket &inTsPacket);
private:
AVCodec *mpDecoder;
AVCodecContext *mpDecoderContext;
AVFrame *mpDecodedFrame;
AVFormatContext *mpFmtContext;
};
VideoDecoder::VideoDecoder()
{
av_register_all();
// FORMAT CONTEXT SETUP
mpFmtContext = avformat_alloc_context();
mpFmtContext->flags = AVFMT_NOFILE;
// ????? WHAT ELSE ???? //
// DECODER SETUP
mpDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
if (!mpDecoder)
{
printf("Could not load decoder\n");
exit(11);
}
mpDecoderContext = avcodec_alloc_context3(NULL);
if (avcodec_open2(mpDecoderContext, mpDecoder, NULL) < 0)
{
printf("Cannot open decoder context\n");
exit(1);
}
mpDecodedFrame = av_frame_alloc();
}
bool
VideoDecoder::rcvTsPacket(AVPacket &inTsPkt)
{
bool ret = true;
if ((av_read_frame(mpFmtContext, &inTsPkt)) < 0)
{
printf("Error in av_read_frame()\n");
ret = false;
}
else
{
// success. Decode the TS packet
int got;
int len = avcodec_decode_video2(mpDecoderContext, mpDecodedFrame, &got, &inTsPkt);
if (len < 0)
ret = false;
if (got)
printf("GOT A DECODED FRAME\n");
}
return ret;
}
int
main(int argc, char **argv)
{
if (argc != 2)
{
printf("Usage: %s tsInFile\n", argv[0]);
exit(1);
}
FILE *tsInFile = fopen(argv[1], "r");
if (!tsInFile)
{
perror("Could not open TS input file");
exit(2);
}
unsigned int tsPktNum = 0;
uint8_t tsBuffer[256];
AVPacket tsPkt;
av_init_packet(&tsPkt);
VideoDecoder vDecoder;
while (!feof(tsInFile))
{
tsPktNum++;
tsPkt.size = 188;
tsPkt.data = tsBuffer;
fread(tsPkt.data, 188, 1, tsInFile);
vDecoder.rcvTsPacket(tsPkt);
}
}
I've got some code snippets that might help you out as I've been working with MPEG-TS also.
Starting with my packet thread which checks each packet against the stream ID's which I've already found and got the codec contexts:
void *FFMPEG::thread_packet_function(void *arg) {
FFMPEG *ffmpeg = (FFMPEG*)arg;
for (int c = 0; c < MAX_PACKETS; c++)
ffmpeg->free_packets[c] = &ffmpeg->packet_list[c];
ffmpeg->packet_pos = MAX_PACKETS;
Audio.start_decoding();
Video.start_decoding();
Subtitle.start_decoding();
while (!ffmpeg->thread_quit) {
if (ffmpeg->packet_pos != 0 &&
Audio.okay_add_packet() &&
Video.okay_add_packet() &&
Subtitle.okay_add_packet()) {
pthread_mutex_lock(&ffmpeg->packet_mutex); // get free packet
AVPacket *pkt = ffmpeg->free_packets[--ffmpeg->packet_pos]; // pre decrement
pthread_mutex_unlock(&ffmpeg->packet_mutex);
if ((av_read_frame(ffmpeg->fContext, pkt)) >= 0) { // success
int id = pkt->stream_index;
if (id == ffmpeg->aud_stream.stream_id) Audio.add_packet(pkt);
else if (id == ffmpeg->vid_stream.stream_id) Video.add_packet(pkt);
else if (id == ffmpeg->sub_stream.stream_id) Subtitle.add_packet(pkt);
else { // unknown packet
av_packet_unref(pkt);
pthread_mutex_lock(&ffmpeg->packet_mutex); // put packet back
ffmpeg->free_packets[ffmpeg->packet_pos++] = pkt;
pthread_mutex_unlock(&ffmpeg->packet_mutex);
//LOGI("Dumping unknown packet, id %d", id);
}
} else {
av_packet_unref(pkt);
pthread_mutex_lock(&ffmpeg->packet_mutex); // put packet back
ffmpeg->free_packets[ffmpeg->packet_pos++] = pkt;
pthread_mutex_unlock(&ffmpeg->packet_mutex);
//LOGI("No packet read");
}
} else { // buffers full so yield
//LOGI("Packet reader on hold: Audio-%d, Video-%d, Subtitle-%d",
// Audio.packet_pos, Video.packet_pos, Subtitle.packet_pos);
usleep(1000);
//sched_yield();
}
}
return 0;
}
Each decoder for audio, video and subtitles have their own threads which receive the packets from the above thread in ring buffers. I've had to separate the decoders into their own threads because CPU usage was increasing when I started using the deinterlace filter.
My video decoder reads the packets from the buffers and when it has finished with the packet sends it back to be unref'd and can be used again. Balancing the packet buffers doesn't take that much time once everything is running.
Here's the snipped from my video decoder:
void *VideoManager::decoder(void *arg) {
LOGI("Video decoder started");
VideoManager *mgr = (VideoManager *)arg;
while (!ffmpeg.thread_quit) {
pthread_mutex_lock(&mgr->packet_mutex);
if (mgr->packet_pos != 0) {
// fetch first packet to decode
AVPacket *pkt = mgr->packets[0];
// shift list down one
for (int c = 1; c < mgr->packet_pos; c++) {
mgr->packets[c-1] = mgr->packets[c];
}
mgr->packet_pos--;
pthread_mutex_unlock(&mgr->packet_mutex); // finished with packets array
int got;
AVFrame *frame = ffmpeg.vid_stream.frame;
avcodec_decode_video2(ffmpeg.vid_stream.context, frame, &got, pkt);
ffmpeg.finished_with_packet(pkt);
if (got) {
#ifdef INTERLACE_ALL
if (!frame->interlaced_frame) mgr->add_av_frame(frame, 0);
else {
if (!mgr->filter_initialised) mgr->init_filter_graph(frame);
av_buffersrc_add_frame_flags(mgr->filter_src_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
int c = 0;
while (true) {
AVFrame *filter_frame = ffmpeg.vid_stream.filter_frame;
int result = av_buffersink_get_frame(mgr->filter_sink_ctx, filter_frame);
if (result == AVERROR(EAGAIN) ||
result == AVERROR(AVERROR_EOF) ||
result < 0)
break;
mgr->add_av_frame(filter_frame, c++);
av_frame_unref(filter_frame);
}
//LOGI("Interlaced %d frames, decode %d, playback %d", c, mgr->decode_pos, mgr->playback_pos);
}
#elif defined(INTERLACE_HALF)
if (!frame->interlaced_frame) mgr->add_av_frame(frame, 0);
else {
if (!mgr->filter_initialised) mgr->init_filter_graph(frame);
av_buffersrc_add_frame_flags(mgr->filter_src_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
int c = 0;
while (true) {
AVFrame *filter_frame = ffmpeg.vid_stream.filter_frame;
int result = av_buffersink_get_frame(mgr->filter_sink_ctx, filter_frame);
if (result == AVERROR(EAGAIN) ||
result == AVERROR(AVERROR_EOF) ||
result < 0)
break;
mgr->add_av_frame(filter_frame, c++);
av_frame_unref(filter_frame);
}
//LOGI("Interlaced %d frames, decode %d, playback %d", c, mgr->decode_pos, mgr->playback_pos);
}
#else
mgr->add_av_frame(frame, 0);
#endif
}
//LOGI("decoded video packet");
} else {
pthread_mutex_unlock(&mgr->packet_mutex);
}
}
LOGI("Video decoder ended");
}
As you can see, I'm using a mutex when passing packets back and forth.
Once a frame has been got I just copy the YUV buffers from the frame for later use into another buffer list. I don't convert the YUV, I use a shader which converts the YUV to RGB on the GPU.
The next snippet adds my decoded frame to my buffer list. This may help understand how to deal with the data.
void VideoManager::add_av_frame(AVFrame *frame, int field_num) {
int y_linesize = frame->linesize[0];
int u_linesize = frame->linesize[1];
int hgt = frame->height;
int y_buffsize = y_linesize * hgt;
int u_buffsize = u_linesize * hgt / 2;
int buffsize = y_buffsize + u_buffsize + u_buffsize;
VideoBuffer *buffer = &buffers[decode_pos];
if (ffmpeg.is_network && playback_pos == decode_pos) { // patched 25/10/16 wlgfx
buffer->used = false;
if (!buffer->data) buffer->data = (char*)mem.alloc(buffsize);
if (!buffer->data) {
LOGI("Dropped frame, allocation error");
return;
}
} else if (playback_pos == decode_pos) {
LOGI("Dropped frame, ran out of decoder frame buffers");
return;
} else if (!buffer->data) {
buffer->data = (char*)mem.alloc(buffsize);
if (!buffer->data) {
LOGI("Dropped frame, allocation error.");
return;
}
}
buffer->y_frame = buffer->data;
buffer->u_frame = buffer->y_frame + y_buffsize;
buffer->v_frame = buffer->y_frame + y_buffsize + u_buffsize;
buffer->wid = frame->width;
buffer->hgt = hgt;
buffer->y_linesize = y_linesize;
buffer->u_linesize = u_linesize;
int64_t pts = av_frame_get_best_effort_timestamp(frame);
buffer->pts = pts;
buffer->buffer_size = buffsize;
double field_add = av_q2d(ffmpeg.vid_stream.context->time_base) * field_num;
buffer->frame_time = av_q2d(ts_stream) * pts + field_add;
memcpy(buffer->y_frame, frame->data[0], (size_t) (buffer->y_linesize * buffer->hgt));
memcpy(buffer->u_frame, frame->data[1], (size_t) (buffer->u_linesize * buffer->hgt / 2));
memcpy(buffer->v_frame, frame->data[2], (size_t) (buffer->u_linesize * buffer->hgt / 2));
buffer->used = true;
decode_pos = (++decode_pos) % MAX_VID_BUFFERS;
//if (field_num == 0) LOGI("Video %.2f, %d - %d",
// buffer->frame_time - Audio.pts_start_time, decode_pos, playback_pos);
}
If there's anything else that I may be able to help with just give me a shout. :-)
EDIT:
The snippet how I open my video stream context which automatically determines the codec, whether it is h264, mpeg2, or another:
void FFMPEG::open_video_stream() {
vid_stream.stream_id = av_find_best_stream(fContext, AVMEDIA_TYPE_VIDEO,
-1, -1, &vid_stream.codec, 0);
if (vid_stream.stream_id == -1) return;
vid_stream.context = fContext->streams[vid_stream.stream_id]->codec;
if (!vid_stream.codec || avcodec_open2(vid_stream.context,
vid_stream.codec, NULL) < 0) {
vid_stream.stream_id = -1;
return;
}
vid_stream.frame = av_frame_alloc();
vid_stream.filter_frame = av_frame_alloc();
}
EDIT2:
This is how I've opened the input stream, whether it be file or URL. The AVFormatContext is the main context for the stream.
bool FFMPEG::start_stream(char *url_, float xtrim, float ytrim, int gain) {
aud_stream.stream_id = -1;
vid_stream.stream_id = -1;
sub_stream.stream_id = -1;
this->url = url_;
this->xtrim = xtrim;
this->ytrim = ytrim;
Audio.volume = gain;
Audio.init();
Video.init();
fContext = avformat_alloc_context();
if ((avformat_open_input(&fContext, url_, NULL, NULL)) != 0) {
stop_stream();
return false;
}
if ((avformat_find_stream_info(fContext, NULL)) < 0) {
stop_stream();
return false;
}
// network stream will overwrite packets if buffer is full
is_network = url.substr(0, 4) == "udp:" ||
url.substr(0, 4) == "rtp:" ||
url.substr(0, 5) == "rtsp:" ||
url.substr(0, 5) == "http:"; // added for wifi broadcasting ability
// determine if stream is audio only
is_mp3 = url.substr(url.size() - 4) == ".mp3";
LOGI("Stream: %s", url_);
if (!open_audio_stream()) {
stop_stream();
return false;
}
if (is_mp3) {
vid_stream.stream_id = -1;
sub_stream.stream_id = -1;
} else {
open_video_stream();
open_subtitle_stream();
if (vid_stream.stream_id == -1) { // switch to audio only
close_subtitle_stream();
is_mp3 = true;
}
}
LOGI("Audio: %d, Video: %d, Subtitle: %d",
aud_stream.stream_id,
vid_stream.stream_id,
sub_stream.stream_id);
if (aud_stream.stream_id != -1) {
LOGD("Audio stream time_base {%d, %d}",
aud_stream.context->time_base.num,
aud_stream.context->time_base.den);
}
if (vid_stream.stream_id != -1) {
LOGD("Video stream time_base {%d, %d}",
vid_stream.context->time_base.num,
vid_stream.context->time_base.den);
}
LOGI("Starting packet and decode threads");
thread_quit = false;
pthread_create(&thread_packet, NULL, &FFMPEG::thread_packet_function, this);
Display.set_overlay_timout(3.0);
return true;
}
EDIT: (constructing an AVPacket)
Construct an AVPacket to send to the decoder...
AVPacket packet;
av_init_packet(&packet);
packet.data = myTSpacketdata; // pointer to the TS packet
packet.size = 188;
You should be able to reuse the packet. And it might need unref'ing.
You must first use the avcodec library to get the compressed frames out of the file. Then you can decode them using avcodec_decode_video2. look at this tutorial http://dranger.com/ffmpeg/
I have found some examples that claims to list all output devices for audio using openAL, however I can only get them to list the currently selected device on OSX (Yosemite, Maverick). I am using a mac and have the default sound card (Built-in Output) as well as an airplay device and some sound flower devices.
(Note: On windows it lists all devices)
I am expecting s = (char *)alcGetString(NULL, ALC_ALL_DEVICES_SPECIFIER); to list at least the default card and the sound flower devices. It returns nothing.
s = (char *)alcGetString(NULL, ALC_DEVICE_SPECIFIER); Returns the currently selected device.
Maybe there is some issue with the devices being virtual? however, selecting soundflower from mac settings as default output first make it show up instead o "Built-in Output". We also tried with an external DAC and got the same behaviour.
list of all available output devices: Built-in Output
list of all available input devices: Built-in Microphone
default output device: Built-in Output
default input device: Built-in Microphone
...
Here is the code:
#ifdef __APPLE__
# include <OpenAL/al.h>
# include <OpenAL/alc.h>
#else
# include <AL/al.h>
# include <AL/alc.h>
# include <AL/alext.h>
#endif
#ifndef AL_VERSION_1_1
# ifdef __APPLE__
# include <OpenAL/altypes.h>
# include <OpenAL/alctypes.h>
#else
# include <AL/altypes.h>
# include <AL/alctypes.h>
# endif
#endif
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <string>
using std::string;
#ifndef ALC_ALL_DEVICES_SPECIFIER
# define ALC_ALL_DEVICES_SPECIFIER 0x1013
#endif
#define MAX_DATA 16
static const int indentation = 4;
static const int maxmimumWidth = 79;
void printExtensions (const char *, char, const char *);
void displayDevices(const char *, const char *);
char *getDeviceName(int, char **);
void testForError(void *, const string&);
void testForALCError(ALCdevice *);
int main(int argc, char **argv)
{
ALCint data[MAX_DATA];
ALCdevice *device = NULL;
ALCcontext *context = NULL;
ALenum error;
char *s;
if (alcIsExtensionPresent(NULL, "ALC_enumeration_EXT") == AL_TRUE)
{
if (alcIsExtensionPresent(NULL, "ALC_enumerate_all_EXT") == AL_FALSE)
s = (char *)alcGetString(NULL, ALC_DEVICE_SPECIFIER);
else
s = (char *)alcGetString(NULL, ALC_ALL_DEVICES_SPECIFIER);
displayDevices("output", s);
s = (char *)alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER);
displayDevices("input", s);
}
s = getDeviceName(argc, argv);
device = alcOpenDevice(s);
testForError(device, "Audio device not available.");
context = alcCreateContext(device, NULL);
testForError(context, "Unable to create a valid context.");
alcMakeContextCurrent(context);
testForALCError(device);
s = (char *)alcGetString(device, ALC_DEFAULT_DEVICE_SPECIFIER);
printf("default output device: %s\n", s);
testForALCError(device);
error = alcIsExtensionPresent(device, "ALC_EXT_capture");
if (error)
{
s = (char *)alcGetString(device, ALC_CAPTURE_DEFAULT_DEVICE_SPECIFIER);
printf("default input device: %s\n", s);
testForALCError(device);
}
printf("capture support: %s\n", (error) ? "yes" : "no");
alcGetIntegerv(device, ALC_FREQUENCY, 1, data);
printf("mixer frequency: %u hz\n", data[0]);
testForALCError(device);
alcGetIntegerv(device, ALC_REFRESH, 1, data+1);
printf("refresh rate : %u hz\n", data[0]/data[1]);
testForALCError(device);
data[0] = 0;
alcGetIntegerv(device, ALC_MONO_SOURCES, 1, data);
error = alcGetError(device);
if (error == AL_NONE) {
printf("supported sources; mono: %u, ", data[0]);
data[0] = 0;
alcGetIntegerv(device, ALC_STEREO_SOURCES, 1, data);
printf("stereo: %u\n", data[0]);
testForALCError(device);
}
printf("ALC version: ");
alcGetIntegerv(device, ALC_MAJOR_VERSION, 1, data);
printf("%i.", *data);
alcGetIntegerv(device, ALC_MINOR_VERSION, 1, data);
printf("%i\n", *data);
testForALCError(device);
s = (char *)alcGetString(device, ALC_EXTENSIONS);
printExtensions ("ALC extensions", ' ', s);
testForALCError(device);
s = (char *)alGetString(AL_VENDOR);
error = alGetError();
if ((error = alGetError()) != AL_NO_ERROR)
printf("Error #%x: %s\n", error, alGetString(error));
else
printf("OpenAL vendor string: %s\n", s);
s = (char *)alGetString(AL_RENDERER);
if ((error = alGetError()) != AL_NO_ERROR)
printf("Error #%x: %s\n", error, alGetString(error));
else
printf("OpenAL renderer string: %s\n", s);
s = (char *)alGetString(AL_VERSION);
if ((error = alGetError()) != AL_NO_ERROR)
printf("Error #%x: %s\n", error, alGetString(error));
else if (!s)
printf("Quering AL_VERSION returned NULL pointer!\n");
else
printf("OpenAL version string: %s\n", s);
s = (char *)alGetString(AL_EXTENSIONS);
printExtensions ("OpenAL extensions", ' ', s);
testForALCError(device);
/* alut testing mechanism */
context = alcGetCurrentContext();
if (context == NULL)
{
printf("Error: no current context\n");
}
else
{
if (alGetError () != AL_NO_ERROR)
{
printf("Alert: AL error on entry\n");
}
else
{
if (alcGetError (alcGetContextsDevice (context)) != ALC_NO_ERROR)
{
printf("Alert: ALC error on entry\n");
}
}
}
/* end of alut test */
if (alcMakeContextCurrent(NULL) == 0)
printf("alcMakeContextCurrent failed.\n");
device = alcGetContextsDevice(context);
alcDestroyContext(context);
testForALCError(device);
if (alcCloseDevice(device) == 0)
printf("alcCloseDevice failed.\n");
return 0;
}
/* -------------------------------------------------------------------------- */
void
printChar (int c, int *width)
{
putchar (c);
*width = (c == '\n') ? 0 : (*width + 1);
}
void
indent (int *width)
{
int i;
for (i = 0; i < indentation; i++)
{
printChar (' ', width);
}
}
void
printExtensions (const char *header, char separator, const char *extensions)
{
int width = 0, start = 0, end = 0;
printf ("%s:\n", header);
if (extensions == NULL || extensions[0] == '\0')
{
return;
}
indent (&width);
while (1)
{
if (extensions[end] == separator || extensions[end] == '\0')
{
if (width + end - start + 2 > maxmimumWidth)
{
printChar ('\n', &width);
indent (&width);
}
while (start < end)
{
printChar (extensions[start], &width);
start++;
}
if (extensions[end] == '\0')
{
break;
}
start++;
end++;
if (extensions[end] == '\0')
{
break;
}
printChar (',', &width);
printChar (' ', &width);
}
end++;
}
printChar ('\n', &width);
}
char *
getCommandLineOption(int argc, char **argv, const string& option)
{
int slen = option.size();
char *rv = 0;
int i;
for (i=0; i<argc; i++)
{
if (strncmp(argv[i], option.c_str(), slen) == 0)
{
i++;
if (i<argc) rv = argv[i];
}
}
return rv;
}
char *
getDeviceName(int argc, char **argv)
{
static char devname[255];
int len = 255;
char *s;
s = getCommandLineOption(argc, argv, "-d");
if (s)
{
strncpy((char *)&devname, s, len);
len -= strlen(s);
s = getCommandLineOption(argc, argv, "-r");
if (s)
{
strncat((char *)&devname, " on ", len);
len -= 4;
strncat((char *)&devname, s, len);
}
s = (char *)&devname;
}
return s;
}
void
displayDevices(const char *type, const char *list)
{
ALCchar *ptr, *nptr;
ptr = (ALCchar *)list;
printf("list of all available %s devices:\n", type);
if (!list)
{
printf("none\n");
}
else
{
nptr = ptr;
while (*(nptr += strlen(ptr)+1) != 0)
{
printf(" %s\n", ptr);
ptr = nptr;
}
printf(" %s\n", ptr);
}
}
void
testForError(void *p, const string& s)
{
if (p == NULL)
{
printf("\nError: %s\n\n", s.c_str());
exit(-1);
}
}
void
testForALCError(ALCdevice *device)
{
ALenum error;
error = alcGetError(device);
if (error != ALC_NO_ERROR)
printf("\nALC Error %x occurred: %s\n", error, alcGetString(device, error));
}
Just to give you ammunition ... here I compiled your above code and give its console output as executed on linux ubuntu 14.04 g++ (Ubuntu 4.9.1-16ubuntu6) 4.9.1
g++ -o openal_list_audio_devices openal_list_audio_devices.cpp -lopenal
stens#kamchatka ~/Dropbox/Documents/code/c++ λ
stens#kamchatka ~/Dropbox/Documents/code/c++ λ ./openal_list_audio_devices
list of all available output devices:
Built-in Audio Analog Stereo
list of all available input devices:
Built-in Audio Analog Stereo
Monitor of Built-in Audio Analog Stereo
default output device: OpenAL Soft
default input device: Built-in Audio Analog Stereo
capture support: yes
mixer frequency: 44100 hz
refresh rate : 1025 hz
supported sources; mono: 255, stereo: 1
ALC version: 1.1
ALC extensions:
ALC_ENUMERATE_ALL_EXT, ALC_ENUMERATION_EXT, ALC_EXT_CAPTURE,
ALC_EXT_DEDICATED, ALC_EXT_disconnect, ALC_EXT_EFX,
ALC_EXT_thread_local_context, ALC_SOFT_loopback
OpenAL vendor string: OpenAL Community
OpenAL renderer string: OpenAL Soft
OpenAL version string: 1.1 ALSOFT 1.15.1
OpenAL extensions:
AL_EXT_ALAW, AL_EXT_DOUBLE, AL_EXT_EXPONENT_DISTANCE, AL_EXT_FLOAT32,
AL_EXT_IMA4, AL_EXT_LINEAR_DISTANCE, AL_EXT_MCFORMATS, AL_EXT_MULAW,
AL_EXT_MULAW_MCFORMATS, AL_EXT_OFFSET, AL_EXT_source_distance_model,
AL_LOKI_quadriphonic, AL_SOFT_buffer_samples, AL_SOFT_buffer_sub_data,
AL_SOFTX_deferred_updates, AL_SOFT_direct_channels, AL_SOFT_loop_points,
AL_SOFT_source_latency
I want to try "userspace" driver(it isn't my code) on Ubuntu. And I have this code:
#include <stdio.h>
#include <string.h>
#include <usb.h>
#include <usbpp.h>
#define DEBUG_LEVEL 0
struct usb_bus *busses,*bus,*dbus;
int main(int argc, char *argv[])
{
struct usb_device *dev;
char *buf;
int n,x,r, ret;
char *ibus, *idev;
char string[64];
int found = 0;
usb_dev_handle *udev;
usb_set_debug(DEBUG_LEVEL);
usb_init(); //initilize the usb library
usb_find_busses();
usb_find_devices();
busses=usb_get_busses();
//use the command line arguments for Bus and Device
if (argc >= 3) {
ibus = argv[1];
idev = argv[2];
}
else { printf("Usage: usbtest <Bus> <Device> [<numBytes>])\n"); exit(1); }
printf("Bus = %s Device = %s\n", ibus, idev);
for (bus=busses; bus && (found == 0); bus=bus->next) { // busses loop
if (strcmp(bus->dirname, ibus) == 0) {
for (dev=bus->devices; dev; dev=dev->next) { // devices loop
if (strcmp(dev->filename, idev) == 0) { dbus=bus; found=1; break; }
}
}
}
if (found == 0) { printf("Unable to find the required device !\nexiting\n"); exit(1); }
printf("Found device\n");
printf("Now we are dealing with device from vendor ID : %d (%x) \n",dev->descriptor.idVendor,dev->descriptor.idVendor);
printf("Trying to open the device...\n");
if (udev=usb_open(dev)) printf("Device opened successfully.\n");
else { printf("Operation failed :-("); exit(1);}
buf=(char*)calloc(1,100);
if (usb_get_driver_np(udev,0,buf,100)) printf("Could not read the driver name :-( %s\n",buf);
else printf("Kernel Using Driver :\n");
// detach the driver from the kernel , seems to be just like rmmod?
if (usb_detach_kernel_driver_np(udev,0)) printf("Error detaching the device :-(\n");
else printf("Device detached successfully from the kernel.\n");
// reserving the device interface for our application, if another driver/software
//is using the device , it will return 'interface busy'
if (r=usb_claim_interface(udev,0)) printf("Interface Claimed !!\n");
printf("Interface Claim Status : %d\n",r);
printf("Device Protocol : %d\n",dev->descriptor.bDeviceProtocol);
printf("Report Length : %d\n",dev->descriptor.bLength);
printf("Decriptor Type : %d\n",dev->descriptor.bDescriptorType);
printf("End Points : %d\n",dev->config->interface->altsetting->bNumEndpoints);
printf("Interface Class : %d\n",dev->config->interface->altsetting->bInterfaceClass);
printf("Protocol : %d\n",dev->config->interface->altsetting->bInterfaceProtocol);
printf("Interface Number: %d\n",dev->config->interface->altsetting->bInterfaceNumber);
printf("Device Filename : %s\n",dev->filename);
printf("Bus Dir Name : %s\n",dbus->dirname);
usb_get_string_simple(udev,dev->descriptor.iManufacturer,string,sizeof(string));
printf("Device Manfucaturer : %s\n",string);
usb_get_string_simple(udev,dev->descriptor.iProduct,string,sizeof(string));
printf("Product Name : %s\n",string);
usb_get_string_simple(udev,dev->descriptor.iSerialNumber,string,sizeof(string));
printf("Device Serial Number: %s\n",string);
printf("End point addresses : 0x%x\n",dev->config->interface->altsetting->endpoint->bEndpointAddress);
int numBytes = 4;
int errCount = 0;
unsigned char endPoint = 0x81;
int i;
int tmp1 = 5;
int tmp2 = 5;
// reset usb device if command line numBytes = 0 and exit
if (argc == 4 && atoi(argv[3]) == 0) { usb_reset(udev); exit(0); }
// use command line value for numBytes if supplied
if (argc == 4) numBytes = atoi(argv[3]);
if (numBytes > 64) numBytes = 64;
while (1) {
for (x=0; x<numBytes; x++) string[x]=0;
// read numBytes bytes using interrupt_read,
r = usb_interrupt_read(udev, endPoint, string, numBytes, 0);
system("clear");
tmp1+=string[1];
tmp2+=string[2];
if (tmp1>=0 && tmp2>=0 && tmp1<=150 && tmp2<=50){
for (i=0;i<tmp2;i++) printf("\n");
for (i=0;i<tmp1;i++) printf(".");
if (string[0]==0) printf ("X");
else if (string[0]==1) printf("L");
else if(string[0]==2) printf("R");
else if(string[0]==3) printf("LR");
else if(string[0]==4) printf("M");
}
else {
if(tmp1<0) tmp1=0;
else if (tmp1>150) tmp1=150;
if (tmp2<0) tmp2=0;
else if (tmp2>50) tmp2=50;
}
if (string[3]==1) printf ("\nSCROLL UP");
if (string[3]==-1) printf ("\nSCROLL DOWN");
printf ("\n(%d, %d)",tmp1,tmp2);
if ( r < 0 ) errCount++;
if (errCount >= 100) break;
usb_clear_halt(udev,endPoint);
}
printf("Closing Device.\n");
usb_release_interface(udev,0);
usb_close(udev);
return EXIT_SUCCESS;
}
In this code are using API from libusb-0.1 - but i don't know excact version of it. Probably - it is libusb-0.1.12 (it have both headers "usb.h" and "usbpp.h", which are using in this code). I have download this version on ubuntu, install it, using
./configure && make && make install && sudo ldconfig
And after that I tried to do
gcc -c mouse.c -w
to compile this code. Bit it doesnt compile. It has different errors in "usbpp.h" (in standart includes like "string" or "list") or smth else..
Can anyone tell me, what i'n doing fundamentally wrong? And how can i try this code?
PS - i try do test this code on ubuntu 9.04 and ubuntu 12>, both as virtual machines. It's my task in university.
usbpp.h is a c++ header file but you run gcc which is a c compiler. Run g++ instead.
I found one another way to solve this problem:
gcc -o mouse ./mouse.cpp -lstdc++ -lusb
I am processing video with opencv, but at the same time I need to play audio and simply control it, like loud or current frame number.
I think I should create a parallel process with ffmpeg, but I don't know how to do so. Can you explain what to do?
Or do you know another solution?
I think ffmpeg should be used to play audio and SDL for video in this case.
After opening the file with OpenCV and processing the frame, you can use OpenCV -> SDL to display it while retrieving the audio frames through ffmpeg and playing them with SDL.
Here is a nice collection of ffmpeg/SDL tutorials!
I also found a nice post that shows how to capture frames from a video file using ffmpeg, store them in OpenCV cv::Mat and display the result in a OpenCV window. But this way you can't play audio since OpenCV doesn't deal with that.
You might be interested in reading this post as well: How to avoid a growing delay with ffmpeg between sound and raw video data ?
EDIT:
I spent the last 4hrs coding a prototype to demonstrate how it's done. This demo reads video frames through OpenCV (so you can process them) and audio through ffmpeg, and SDL is used to play both! There are 2 limitations in this demo you must be aware: 1 - it assumes you are working with an OpenCV image packed as BGR (24bits), and 2 - audio and video are not being sync! Yes, I left have some work for you to do (yeeeey). But don't panic, page 6 has some ideas!
It's important to sync audio and video because you will be doing some processing on the frames, and that will certainly make the video and audio go out of sync real fast since they are being played independently of each other.
The ffmpeg tutorials I suggested above are very very important to understand the code, a lot of code from this demo came from there. They show how to deal with SDL, and how to read packets of audio/video streams.
#include <highgui.h>
#include <cv.h>
extern "C"
{
#include <SDL.h>
#include <SDL_thread.h>
#include <avcodec.h>
#include <avformat.h>
}
#include <iostream>
#include <stdio.h>
//#include <malloc.h>
using namespace cv;
#define SDL_AUDIO_BUFFER_SIZE 1024
typedef struct PacketQueue
{
AVPacketList *first_pkt, *last_pkt;
int nb_packets;
int size;
SDL_mutex *mutex;
SDL_cond *cond;
} PacketQueue;
PacketQueue audioq;
int audioStream = -1;
int videoStream = -1;
int quit = 0;
SDL_Surface* screen = NULL;
SDL_Surface* surface = NULL;
AVFormatContext* pFormatCtx = NULL;
AVCodecContext* aCodecCtx = NULL;
AVCodecContext* pCodecCtx = NULL;
void show_frame(IplImage* img)
{
if (!screen)
{
screen = SDL_SetVideoMode(img->width, img->height, 0, 0);
if (!screen)
{
fprintf(stderr, "SDL: could not set video mode - exiting\n");
exit(1);
}
}
// Assuming IplImage packed as BGR 24bits
SDL_Surface* surface = SDL_CreateRGBSurfaceFrom((void*)img->imageData,
img->width,
img->height,
img->depth * img->nChannels,
img->widthStep,
0xff0000, 0x00ff00, 0x0000ff, 0
);
SDL_BlitSurface(surface, 0, screen, 0);
SDL_Flip(screen);
}
void packet_queue_init(PacketQueue *q)
{
memset(q, 0, sizeof(PacketQueue));
q->mutex = SDL_CreateMutex();
q->cond = SDL_CreateCond();
}
int packet_queue_put(PacketQueue *q, AVPacket *pkt)
{
AVPacketList *pkt1;
if (av_dup_packet(pkt) < 0)
{
return -1;
}
//pkt1 = (AVPacketList*) av_malloc(sizeof(AVPacketList));
pkt1 = (AVPacketList*) malloc(sizeof(AVPacketList));
if (!pkt1) return -1;
pkt1->pkt = *pkt;
pkt1->next = NULL;
SDL_LockMutex(q->mutex);
if (!q->last_pkt)
q->first_pkt = pkt1;
else
q->last_pkt->next = pkt1;
q->last_pkt = pkt1;
q->nb_packets++;
q->size += pkt1->pkt.size;
SDL_CondSignal(q->cond);
SDL_UnlockMutex(q->mutex);
return 0;
}
static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block)
{
AVPacketList *pkt1;
int ret;
SDL_LockMutex(q->mutex);
for (;;)
{
if( quit)
{
ret = -1;
break;
}
pkt1 = q->first_pkt;
if (pkt1)
{
q->first_pkt = pkt1->next;
if (!q->first_pkt)
q->last_pkt = NULL;
q->nb_packets--;
q->size -= pkt1->pkt.size;
*pkt = pkt1->pkt;
//av_free(pkt1);
free(pkt1);
ret = 1;
break;
}
else if (!block)
{
ret = 0;
break;
}
else
{
SDL_CondWait(q->cond, q->mutex);
}
}
SDL_UnlockMutex(q->mutex);
return ret;
}
int audio_decode_frame(AVCodecContext *aCodecCtx, uint8_t *audio_buf, int buf_size)
{
static AVPacket pkt;
static uint8_t *audio_pkt_data = NULL;
static int audio_pkt_size = 0;
int len1, data_size;
for (;;)
{
while (audio_pkt_size > 0)
{
data_size = buf_size;
len1 = avcodec_decode_audio2(aCodecCtx, (int16_t*)audio_buf, &data_size,
audio_pkt_data, audio_pkt_size);
if (len1 < 0)
{
/* if error, skip frame */
audio_pkt_size = 0;
break;
}
audio_pkt_data += len1;
audio_pkt_size -= len1;
if (data_size <= 0)
{
/* No data yet, get more frames */
continue;
}
/* We have data, return it and come back for more later */
return data_size;
}
if (pkt.data)
av_free_packet(&pkt);
if (quit) return -1;
if (packet_queue_get(&audioq, &pkt, 1) < 0) return -1;
audio_pkt_data = pkt.data;
audio_pkt_size = pkt.size;
}
}
void audio_callback(void *userdata, Uint8 *stream, int len)
{
AVCodecContext *aCodecCtx = (AVCodecContext *)userdata;
int len1, audio_size;
static uint8_t audio_buf[(AVCODEC_MAX_AUDIO_FRAME_SIZE * 3) / 2];
static unsigned int audio_buf_size = 0;
static unsigned int audio_buf_index = 0;
while (len > 0)
{
if (audio_buf_index >= audio_buf_size)
{
/* We have already sent all our data; get more */
audio_size = audio_decode_frame(aCodecCtx, audio_buf, sizeof(audio_buf));
if(audio_size < 0)
{
/* If error, output silence */
audio_buf_size = 1024; // arbitrary?
memset(audio_buf, 0, audio_buf_size);
}
else
{
audio_buf_size = audio_size;
}
audio_buf_index = 0;
}
len1 = audio_buf_size - audio_buf_index;
if (len1 > len)
len1 = len;
memcpy(stream, (uint8_t *)audio_buf + audio_buf_index, len1);
len -= len1;
stream += len1;
audio_buf_index += len1;
}
}
void setup_ffmpeg(char* filename)
{
if (av_open_input_file(&pFormatCtx, filename, NULL, 0, NULL) != 0)
{
fprintf(stderr, "FFmpeg failed to open file %s!\n", filename);
exit(-1);
}
if (av_find_stream_info(pFormatCtx) < 0)
{
fprintf(stderr, "FFmpeg failed to retrieve stream info!\n");
exit(-1);
}
// Dump information about file onto standard error
dump_format(pFormatCtx, 0, filename, 0);
// Find the first video stream
int i = 0;
for (i; i < pFormatCtx->nb_streams; i++)
{
if (pFormatCtx->streams[i]->codec->codec_type == CODEC_TYPE_VIDEO && videoStream < 0)
{
videoStream = i;
}
if (pFormatCtx->streams[i]->codec->codec_type == CODEC_TYPE_AUDIO && audioStream < 0)
{
audioStream = i;
}
}
if (videoStream == -1)
{
fprintf(stderr, "No video stream found in %s!\n", filename);
exit(-1);
}
if (audioStream == -1)
{
fprintf(stderr, "No audio stream found in %s!\n", filename);
exit(-1);
}
// Get a pointer to the codec context for the audio stream
aCodecCtx = pFormatCtx->streams[audioStream]->codec;
// Set audio settings from codec info
SDL_AudioSpec wanted_spec;
wanted_spec.freq = aCodecCtx->sample_rate;
wanted_spec.format = AUDIO_S16SYS;
wanted_spec.channels = aCodecCtx->channels;
wanted_spec.silence = 0;
wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE;
wanted_spec.callback = audio_callback;
wanted_spec.userdata = aCodecCtx;
SDL_AudioSpec spec;
if (SDL_OpenAudio(&wanted_spec, &spec) < 0)
{
fprintf(stderr, "SDL_OpenAudio: %s\n", SDL_GetError());
exit(-1);
}
AVCodec* aCodec = avcodec_find_decoder(aCodecCtx->codec_id);
if (!aCodec)
{
fprintf(stderr, "Unsupported codec!\n");
exit(-1);
}
avcodec_open(aCodecCtx, aCodec);
// audio_st = pFormatCtx->streams[index]
packet_queue_init(&audioq);
SDL_PauseAudio(0);
// Get a pointer to the codec context for the video stream
pCodecCtx = pFormatCtx->streams[videoStream]->codec;
// Find the decoder for the video stream
AVCodec* pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
if (pCodec == NULL)
{
fprintf(stderr, "Unsupported codec!\n");
exit(-1); // Codec not found
}
// Open codec
if (avcodec_open(pCodecCtx, pCodec) < 0)
{
fprintf(stderr, "Unsupported codec!\n");
exit(-1); // Could not open codec
}
}
int main(int argc, char* argv[])
{
if (argc < 2)
{
std::cout << "Usage: " << argv[0] << " <video>" << std::endl;
return -1;
}
av_register_all();
// Init SDL
if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER))
{
fprintf(stderr, "Could not initialize SDL - %s\n", SDL_GetError());
return -1;
}
// Init ffmpeg and setup some SDL stuff related to Audio
setup_ffmpeg(argv[1]);
VideoCapture cap(argv[1]); // open the default camera
if (!cap.isOpened()) // check if we succeeded
{
std::cout << "Failed to load file!" << std::endl;
return -1;
}
AVPacket packet;
while (av_read_frame(pFormatCtx, &packet) >= 0)
{
if (packet.stream_index == videoStream)
{
// Actually this is were SYNC between audio/video would happen.
// Right now I assume that every VIDEO packet contains an entire video frame, and that's not true. A video frame can be made by multiple packets!
// But for the time being, assume 1 video frame == 1 video packet,
// so instead of reading the frame through ffmpeg, I read it through OpenCV.
Mat frame;
cap >> frame; // get a new frame from camera
// do some processing on the frame, either as a Mat or as IplImage.
// For educational purposes, applying a lame grayscale conversion
IplImage ipl_frame = frame;
for (int i = 0; i < ipl_frame.width * ipl_frame.height * ipl_frame.nChannels; i += ipl_frame.nChannels)
{
ipl_frame.imageData[i] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //B
ipl_frame.imageData[i+1] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //G
ipl_frame.imageData[i+2] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //R
}
// Display it on SDL window
show_frame(&ipl_frame);
av_free_packet(&packet);
}
else if (packet.stream_index == audioStream)
{
packet_queue_put(&audioq, &packet);
}
else
{
av_free_packet(&packet);
}
SDL_Event event;
SDL_PollEvent(&event);
switch (event.type)
{
case SDL_QUIT:
SDL_FreeSurface(surface);
SDL_Quit();
break;
default:
break;
}
}
// the camera will be deinitialized automatically in VideoCapture destructor
// Close the codec
avcodec_close(pCodecCtx);
// Close the video file
av_close_input_file(pFormatCtx);
return 0;
}
On my Mac I compiled it with:
g++ ffmpeg_snd.cpp -o ffmpeg_snd -D_GNU_SOURCE=1 -D_THREAD_SAFE -I/usr/local/include/opencv -I/usr/local/include -I/usr/local/include/SDL -Wl,-framework,Cocoa -L/usr/local/lib -lopencv_core -lopencv_imgproc -lopencv_highgui -lopencv_ml -lopencv_video -lopencv_features2d -lopencv_calib3d -lopencv_objdetect -lopencv_contrib -lopencv_legacy -lopencv_flann -lSDLmain -lSDL -L/usr/local/lib -lavfilter -lavcodec -lavformat -I/usr/local/Cellar/ffmpeg/HEAD/include/libavcodec -I/usr/local/Cellar/ffmpeg/HEAD/include/libavformat