I am writing software to play videos that have been recorded from NVR. I have completed most of the work, but there is one more feature that allows the user to change the play speed such as 0.5x, 2x, 4x, 8x ...
I searched the internet all day and still couldn't find any suggestions. Here is my summary code below.
auto pFormatCtx = avformat_alloc_context();
av_dict_set_int(&opts, "rw_timeout", 5000000, 0);
av_dict_set_int(&opts, "tcp_nodelay", 1, 0);
av_dict_set_int(&opts, "stimeout", 10000000, 0);
av_dict_set(&opts, "user_agent", "Mozilla/5.0", 0);
av_dict_set(&opts, "rtsp_transport", "tcp", 0);
av_dict_set(&opts, "rtsp_flags", "prefer_tcp", 0);
av_dict_set_int(&opts, "buffer_size", BUFSIZE, 0);
int err = avformat_open_input(&pFormatCtx, fullRtspUri, NULL, &opts);
if(err < 0)
return;
err = avformat_find_stream_info(pFormatCtx, NULL);
if (err < 0)
return;
pFormatCtx->flags |= AVFMT_FLAG_NONBLOCK;
pFormatCtx->flags |= AVFMT_FLAG_DISCARD_CORRUPT;
pFormatCtx->flags |= AVFMT_FLAG_NOBUFFER;
av_dump_format(pFormatCtx, 0, fullRtspUri, 0);
int videoStreamInd = -1;
for (int i = 0; i < pFormatCtx->nb_streams; i++)
{
AVStream* stream = pFormatCtx->streams[i];
if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
{
if (videoStreamInd == -1)
{
videoStreamInd = i;
break;
}
}
}
if (videoStreamInd == -1)
return;
auto videoStream = pFormatCtx->streams[videoStreamInd];
isRunning = true;
while(isRunning)
{
ret = av_read_frame(pFormatCtx, avPacket);
if (ret < 0)
return;
if (avPacket->stream_index != videoStreamInd)
continue;
//Code for render process here............
}
I have read through this NVR API documentation and see support for 2x, 4x speed play as below
Play in 2× Speed:
PLAY rtsp://10.17.133.46:554/ISAPI/streaming/tracks/101?starttime=20170313T230652Z&endtime=20170314T025706Z RTSP/1.0
CSeq:6
Authorization: Digest username="admin",
realm="4419b66d2485",
nonce="a0ecd9b1586ff9461f02f910035d0486",
uri="rtsp://10.17.133.46:554/ISAPI/streaming/tracks/101?starttime=20170313T230652Z&endtime=20170314T025706Z",
response="fb986d385a7d839052ec4f0b2b70c631"
Session:2049381566;timeout=60
Scale:2.000
User-Agent:NKPlayer-1.00.00.081112
RTSP/1.0 200 OK
CSeq: 6
Session: 2049381566
Scale: 2.000
RTP-Info: url=trackID=1;seq=1,url=trackID=2;seq=1
Date: Tue, Mar 14 2017 10:57:24 GMT
How to play RTSP video with speeds of 0.5x, 2x, 4x ...?
Everyone who can assist me in this case, I am very grateful.
Somewhere in the "Code for render process here..." you should have code that figures out the delay between presentation/display time for each frame. Whatever that delay is, half it to play twice as fast, double it to play twice as slow and so on. You say this is playing from an NVR, which indicates the stream is probably local and accessed as a media file on disk? If so, it's the speed of the disk and local network connection to the NVR that will limit your ability to play as fast as you want. With my own ffmpeg player code, reading a media file from a M.2 NVMe disk can easily reach 400-700 fps by removing any delay between frames.
Related
I have a webcam connected to beaglebone via usb. I am coding in c++ and my goal is to capture raw UNCOMPRESSED picture from the webcam.
Firstly i checked what formats are supported via command v4l2-ctl --list-formats and the result was:
Index : 0
Type : Video Capture
Pixel Format: 'MJPG' (compressed)
Name : Motion-JPEG
Index : 1
Type : Video Capture
Pixel Format: 'YUYV'
Name : YUYV 4:2:2
So from this I assume it has to be possible to get an uncompressed picture if i try to use YUYV format.
Knowing this I started writing a program in c++. I successfully written a program to capture a compressed picture, but when trying to capture using format YUYV it doesnt work and i really need some help to get this done.
Here is my code:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <errno.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <linux/videodev2.h>
#include <libv4l2.h>
template <typename typeXX>
void clear_memmory(typeXX* x) {
memset(x, 0, sizeof(*x));
}
void xioctl(int cd, int request, void *arg){
int response;
do{
//ensures we get the correct response.
response = v4l2_ioctl(cd, request, arg);
}
while (response == -1 && ((errno == EINTR) || (errno == EAGAIN)));
if (response == -1) {
fprintf(stderr, "error %d, %s\n", errno, strerror(errno));
exit(EXIT_FAILURE);
}
}
struct LMSBBB_buffer{
void* start;
size_t length;
};
int main(){
const char* dev_name = "/dev/video0";
int width=1920;
int height=1080;
int fd = v4l2_open(dev_name, O_RDWR | O_NONBLOCK, 0);
struct v4l2_format format = {0};
format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
format.fmt.pix.width = width;
format.fmt.pix.height = height;
format.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB24;//V4L2_PIX_FMT_YUYV //V4L2_PIX_FMT_RGB24
format.fmt.pix.field = V4L2_FIELD_NONE; //V4L2_FIELD_NONE
xioctl(fd, VIDIOC_S_FMT, &format);
printf("Device initialized.\n");
///request buffers
struct v4l2_requestbuffers req = {0};
req.count = 2;
req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
req.memory = V4L2_MEMORY_MMAP;
xioctl(fd, VIDIOC_REQBUFS, &req);
printf("Buffers requested.\n");
///mapping buffers
struct v4l2_buffer buf;
LMSBBB_buffer* buffers;
unsigned int i;
buffers = (LMSBBB_buffer*) calloc(req.count, sizeof(*buffers));
for (i = 0; i < req.count; i++) {
clear_memmory(&(buf));
(buf).type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
(buf).memory = V4L2_MEMORY_MMAP;
(buf).index = i;
xioctl(fd, VIDIOC_QUERYBUF, &buf);
buffers[i].length = (buf).length;
printf("A buff has a len of: %i\n",buffers[i].length);
buffers[i].start = v4l2_mmap(NULL, (buf).length, PROT_READ | PROT_WRITE, MAP_SHARED,fd, (buf).m.offset);
if (MAP_FAILED == buffers[i].start) {
perror("Can not map the buffers.");
exit(EXIT_FAILURE);
}
}
printf("Buffers mapped.\n");
for (i = 0; i < req.count; i++) {
clear_memmory(&(buf));
(buf).type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
(buf).memory = V4L2_MEMORY_MMAP;
(buf).index = i;
ioctl(fd,VIDIOC_QBUF, &(buf));
}
enum v4l2_buf_type type;
type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ioctl(fd,VIDIOC_STREAMON, &type);
printf("buffers queued and streaming.\n");
int pic_count=0;
///CAPTURE
fd_set fds;
struct timeval tv;
int r;
char out_name[256];
FILE* fout;
do {
FD_ZERO(&fds);
FD_SET(fd, &fds);
// Timeout.
tv.tv_sec = 2;
tv.tv_usec = 0;
r = select(fd + 1, &fds, NULL, NULL, &tv);
} while ((r == -1 && (errno = EINTR)));
if (r == -1) {
perror("select");
exit(EXIT_FAILURE);
}
clear_memmory(&(buf));
(buf).type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
(buf).memory = V4L2_MEMORY_MMAP;
xioctl(fd,VIDIOC_DQBUF, &(buf));
printf("Buff index: %i\n",(buf).index);
sprintf(out_name, "image%03d.ppm",pic_count);
fout = fopen(out_name, "w");
if (!fout) {
perror("Cannot open image");
exit(EXIT_FAILURE);
}
fprintf(fout, "P6\n%d %d 255\n",width, height);
fwrite(buffers[(buf).index].start, (buf).bytesused, 1, fout);
fclose(fout);
pic_count++;
clear_memmory(&(buf));
(buf).type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
(buf).memory = V4L2_MEMORY_MMAP;
xioctl(fd,VIDIOC_DQBUF, &(buf));
printf("Buff index: %i\n",(buf).index);
sprintf(out_name, "image%03d.ppm",pic_count);
fout = fopen(out_name, "w");
if (!fout) {
perror("Cannot open image");
exit(EXIT_FAILURE);
}
fprintf(fout, "P6\n%d %d 255\n",width, height);
fwrite(buffers[(buf).index].start, (buf).bytesused, 1, fout);
fclose(fout);
pic_count++;
///xioctl(fd,VIDIOC_QBUF, &(buf));
return 0;
}
in line 50, i can choose the format between V4L2_PIX_FMT_YUYV and V4L2_PIX_FMT_RGB24.
for V4L2_PIX_FMT_RGB24 i get the picture, but when using V4L2_PIX_FMT_YUYV I get this error:
libv4l2: error dequeuing buf: Resource temporarily unavailable
libv4l2: error dequeuing buf: Resource temporarily unavailable
libv4l2: error dequeuing buf: Resource temporarily unavailable
libv4l2: error dequeuing buf: Resource temporarily unavailable
libv4l2: error dequeuing buf: Resource temporarily unavailable
the error lines goes for ever until i end the program manually.
Does anyone have an idea what to do? I spent over 2 weeks on this and i can't move anywhere from here. I would really appreciate any advice.
From what I see you are requesting a FullHD (1920x1080) buffer in YUYV format from a camera. You did not mention the camera type/model/specs, but if it is a generic USB-attached hardware most likely you will not get a raw FullHD YUYV buffer as an output, only the MJPEG one (which you can decode to YUV, if you hack around with libjpeg) or the decoded RGB buffer (which is pretty much the decoded MJPEG with YUV->RGB conversion) which is not mmapped.
The exact list of formats with framerates can be requested by this command, which would probably tell you it does not provide a 1920x1080 YUYV, only something smaller, like 640x480:
v4l2-ctl --list-formats
If you need video processing with "true" zero-copy access to raw YUYV camera frames, you need direct access to hardware and that specific hardware in the first place. Once you have the USB interface between your software and the camera, you get an extra indirection and that means the speed goes down. Think for a moment, the YUYV frame at 1920x1080 takes up approximately 4 Megabytes of memory. At 30 FPS this is 120 Megabytes (or 960 Megabits) per second of bus throughput. If you have a USB2.0 camera, there is just no bandwidth to support this (thus the need for MJPEG). Even at 15FPS this is 480 Megabits, not counting the USB latency and protocol overhead.
To provide some "actionable feedback" I would advice to first concentrate on the algorithms (probably, you just don't want to loose the processing speed at the very first step) which you want to apply to the image. Don't hesitate to use OpenCV for camera input and basic image processing, later you can switch to some hardware interface and hand-written algorithms.
The easier way of getting raw frames would be to use Android's camera interface and try to process the incoming frames with GLSL shaders using the GL_TEXTURE_EXTERNAL_OES extension, about which there information and code samples available. There you can connect GL textures to AHardwareBuffer instances and then use AHardwareBuffer_lock function to get raw pointers. The exact supported formats also may vary across the hardware, so do not expect this to be super-easy.
I've recently had a similar issue. In my case the camera driver needed the VIDIOC_S_PARM ioctl in order to set the frame rate and initialize the camera for the selected capture mode.
You can try to add this code after the VIDIOC_S_FMT and see if it works for you as well:
struct v4l2_streamparm streamparam;
memset(&streamparam, 0, sizeof(streamparam));
streamparam.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
xioctl(fd, VIDIOC_G_PARM, &streamparam);
streamparam.parm.capture.timeperframe.numerator = 1;
streamparam.parm.capture.timeperframe.denominator = 5;
xioctl(fd, VIDIOC_S_PARM, &streamparam);
I have 4 sounds. I need play sound 1, when it finishes, automatically play sound 2; when sound 2 finishes, automatically play sound 3. Soun 3 finishes, play sound 4.... I'm using SDL Mixer 2.0, no SDL Sound...Is there a way?
int main() {
int frequencia = 22050;
Uint16 formato = AUDIO_S16SYS;
int canal = 2; // 1 mono; 2 = stereo;
int buffer = 4096;
Mix_OpenAudio(frequencia, formato, canal, buffer);
Mix_Chunk* sound_1;
Mix_Chunk* sound_2;
Mix_Chunk* sound_3;
Mix_Chunk* sound_4;
som_1 = Mix_LoadWAV("D:\\sound1.wav");
som_2 = Mix_LoadWAV("D:\\sound1.wav");
som_3 = Mix_LoadWAV("D:\\sound1.wav");
som_4 = Mix_LoadWAV("D:\\sound1.wav");
Mix_PlayChannel(-1, sound_1, 0);
Mix_PlayChannel(1, sound_2, 0);
Mix_PlayChannel(2, sound_3, 0);
Mix_PlayChannel(3, sound_4, 0);
return 0;
}
Check in a loop whether the channel is still playing using Mix_Playing(), and add a delay using SDL_Delay() to prevent the loop from consuming all available CPU time.
(In this example, I changed your first call to Mix_PlayChannel() from -1 to 1.)
Mix_PlayChannel(1, sound_1, 0);
while (Mix_Playing(1) != 0) {
SDL_Delay(200); // wait 200 milliseconds
}
Mix_PlayChannel(2, sound_2, 0);
while (Mix_Playing(2) != 0) {
SDL_Delay(200); // wait 200 milliseconds
}
// etc.
You should probably wrap that into a function instead so that you don't repeat what is basically the same code over and over again:
void PlayAndWait(int channel, Mix_Chunk* chunk, int loops)
{
channel = Mix_PlayChannel(channel, chunk, loops);
if (channel < 0) {
return; // error
}
while (Mix_Playing(channel) != 0) {
SDL_Delay(200);
}
}
// ...
PlayAndWait(-1, sound_1, 0);
PlayAndWait(1, sound_2, 0);
PlayAndWait(2, sound_3, 0);
PlayAndWait(3, sound_3, 0);
I'm using FFMPEG to decode H264 (or H265) RTSP Stream.
My system have 2 software: Server and Client
Server: Read frames from RTSP stream --> Forward frames to Client
Client: Receive frames from Server --> Decode --> Render
I have implemented and it worked ok, but there is a case make my system work not good. That is when internet from Server - Client is slow, frames can not transfer real-time to Client.
In present, I deal with this issue by Skip some frames (not send to Client) when the Queue is reached limit of count. The following is my summary code
//At Server Software (include 2 threads A and B)
//Thread A: Read AVPacket and forward to Client
while(true)
{
AVPacket packet;
av_init_packet(&packet);
packet.size = 0;
packet.data = NULL;
int ret = AVERROR(EAGAIN);
while (AVERROR(EAGAIN) == ret)
ret = av_read_frame(pFormatCtx, &packet);
if(packet.size > 0)
{
if(mySendQueue.count < 120) //limit 120 packet in queue
mySendQueue.Enqueue(packet); ////Thread B will read from this queue, to send packets to Client via TCP socket
else
;//SkipThisFrame ***: No send
}
}
//Thread B: Send To Client via TCP Socket
While(true)
{
AVPacket packet;
if(mySendQueue.Dequeue(packet))
{
SendPacketToClient(packet);
}
}
//At Server Software : Receive AVPacket from Server --> Decode --> Render
While(true)
{
AVPacket packet;
AVFrame frame;
ReadPacketFromServer(packet);
if (av_decode_asyn(pCodecCtx, &frame, &frameFinished, &packet) == RS_OK)
{
if (frameFinished)
{
RenderFrame(frame);
}
}
}
UINT32 __clrcall av_decode_asyn(AVCodecContext *pCodecCtx, AVFrame *frame, int *frameFinished, AVPacket *packet)
{
int ret = -1;
*frameFinished = 0;
if (packet)
{
ret = avcodec_send_packet(pCodecCtx, packet);
// In particular, we don't expect AVERROR(EAGAIN), because we read all
// decoded frames with avcodec_receive_frame() until done.
if (ret < 0 && ret != AVERROR_EOF)
return RS_NOT_OK;
}
ret = avcodec_receive_frame(pCodecCtx, frame);
if (ret < 0 && ret != AVERROR(EAGAIN))
{
return RS_NOT_OK;
}
if (ret >= 0)
*frameFinished = 1;
return RS_OK;
}
My question is focus in line of code SkipThisFrame ***, this algorithm skip frame continuously, so it maybe make the decoder on Client occur unexpectedly error or Crash?
And when skip frame like that, make Client Render frames is not normally?
And someone call show me the proper algorithm to skip frames in my case?
Thank you very much!
I have a brief read on doc of AVPacket, it says:
For video, it should typically contain one compressed frame.
Theoretically you cannot skip frames for a compressed video stream, as most frames do not contain complete information about that frame's image, but only contain changes compared with some previous frames. So if you skip a frame, it is probable that many trailing decoded frames won't contain correct result (until next key frame flushes whole image).
"My question is focus in line of code SkipThisFrame ***, this algorithm
skip frame continuously, so it maybe make the decoder on Client occur
unexpectedly error or Crash?"
One thing I notice is wrong...
Your While(true) statements also need a break; to stop, otherwise they will run forever, blocking other functions and causing the system to crash. Think about it, you say "While the loop is true do X-Y-Z instructions" but you never say when to stop (eg: break out of this While loop to do next instructions). Computer is stuck doing first While loop only and also repeating that to infinity...
Try setting up like this:
//At Server Software (include 2 threads A and B)
//Thread A: Read AVPacket and forward to Client
while(true)
{
AVPacket packet;
av_init_packet(&packet);
packet.size = 0;
packet.data = NULL;
int ret = AVERROR(EAGAIN);
while (AVERROR(EAGAIN) == ret) { ret = av_read_frame(pFormatCtx, &packet); }
if(packet.size > 0)
{
if(mySendQueue.count < 120) //limit 120 packet in queue
{
mySendQueue.Enqueue(packet); ////Thread B will read from this queue, to send packets to Client via TCP socket
}
//else { } //no need for ELSE if doing nothing... //SkipThisFrame ***: No send
}
break; //stop this part and move to "Thead B"
}
//Thread B: Send To Client via TCP Socket
While(true)
{
AVPacket packet;
if( mySendQueue.Dequeue(packet) )
{ SendPacketToClient(packet); break; }
}
//At Server Software : Receive AVPacket from Server --> Decode --> Render
While(true)
{
AVPacket packet; AVFrame frame;
ReadPacketFromServer(packet);
if (av_decode_asyn(pCodecCtx, &frame, &frameFinished, &packet) == RS_OK)
{
if (frameFinished) { RenderFrame(frame); break; }
}
}
UINT32 __clrcall av_decode_asyn(AVCodecContext *pCodecCtx, AVFrame *frame, int *frameFinished, AVPacket *packet)
{
int ret = -1;
*frameFinished = 0;
if (packet)
{
ret = avcodec_send_packet(pCodecCtx, packet);
// In particular, we don't expect AVERROR(EAGAIN), because we read all
// decoded frames with avcodec_receive_frame() until done.
if (ret < 0 && ret != AVERROR_EOF)
return RS_NOT_OK;
}
ret = avcodec_receive_frame(pCodecCtx, frame);
if (ret < 0 && ret != AVERROR(EAGAIN))
{
return RS_NOT_OK;
}
if (ret >= 0)
*frameFinished = 1;
return RS_OK;
}
Hope it helps. Let me know of results / errors.
I wrote some C++ code that uses ffmpeg to encode a video. I'm having two strange issues:
The final video is always missing 1 frame. That is, if I have it encode 10 frames the final video only has 9 (at least that's what ffprobe -show_frames -pretty $VIDEO | grep -F '[FRAME]' | wc -l tells me.
The final video plays fine in some players (mpv and vlc) but not in Quicktime. Quicktime just shows a completely black screen.
My code is roughly this (modified a bit to remove types that are unique to our code base):
First, I open the video file, write the headers and initialize things:
template <class PtrT>
using UniquePtrWithDeleteFunction = std::unique_ptr<PtrT, std::function<void (PtrT*)>>;
std::unique_ptr<FfmpegEncodingFrameSink> FfmpegEncodingFrameSink::Create(
const std::string& dest_url) {
AVFormatContext* tmp_format_ctxt;
auto alloc_format_res = avformat_alloc_output_context2(&tmp_format_ctxt, nullptr, "mp4", dest_url.c_str());
if (alloc_format_res < 0) {
throw FfmpegException("Error opening output file.");
}
auto format_ctxt = UniquePtrWithDeleteFunction<AVFormatContext>(
tmp_format_ctxt, CloseAvFormatContext);
AVStream* out_stream_video = avformat_new_stream(format_ctxt.get(), nullptr);
if (out_stream_video == nullptr) {
throw FfmpegException("Could not create outputstream");
}
auto codec_context = GetCodecContext(options);
out_stream_video->time_base = codec_context->time_base;
auto ret = avcodec_parameters_from_context(out_stream_video->codecpar, codec_context.get());
if (ret < 0) {
throw FfmpegException("Failed to copy encoder parameters to outputstream");
}
if (!(format_ctxt->oformat->flags & AVFMT_NOFILE)) {
ret = avio_open(&format_ctxt->pb, dest_url.c_str(), AVIO_FLAG_WRITE);
if (ret < 0) {
throw VideoDecodeException("Could not open output file: " + dest_url);
}
}
ret = avformat_init_output(format_ctxt.get(), nullptr);
if (ret < 0) {
throw FfmpegException("Unable to initialize the codec.");
}
ret = avformat_write_header(format_ctxt.get(), nullptr);
if (ret < 0) {
throw FfmpegException("Error occurred writing format header");
}
return std::unique_ptr<FfmpegEncodingFrameSink>(
new FfmpegEncodingFrameSink(std::move(format_ctxt), std::move(codec_context)));
}
Then, every time I get a new frame to encode I pass it to this function (the frames are being decoded via ffmpeg from another mp4 file which Quicktime plays just fine):
// If frame == nullptr then we're done and we're just flushing the encoder
// otherwise encode an actual frame
void FfmpegEncodingFrameSink::EncodeAndWriteFrame(
const AVFrame* frame) {
auto ret = avcodec_send_frame(codec_ctxt_.get(), frame);
if (ret < 0) {
throw FfmpegException("Error encoding the frame.");
}
AVPacket enc_packet;
enc_packet.data = nullptr;
enc_packet.size = 0;
av_init_packet(&enc_packet);
do {
ret = avcodec_receive_packet(codec_ctxt_.get(), &enc_packet);
if (ret == AVERROR(EAGAIN)) {
CHECK(frame != nullptr);
break;
} else if (ret == AVERROR_EOF) {
CHECK(frame == nullptr);
break;
} else if (ret < 0) {
throw FfmpegException("Error putting the encoded frame into the packet.");
}
assert(ret == 0);
enc_packet.stream_index = 0;
LOG(INFO) << "Writing packet to stream.";
av_interleaved_write_frame(format_ctxt_.get(), &enc_packet);
av_packet_unref(&enc_packet);
} while (ret == 0);
}
Finally, in my destructor I close everything up like so:
FfmpegEncodingFrameSink::~FfmpegEncodingFrameSink() {
// Pass a nullptr to EncodeAndWriteFrame so it flushes the encoder
EncodeAndWriteFrame(nullptr);
// write mp4 trailer
av_write_trailer(format_ctxt_.get());
}
If I run this passing n frames to EncodeAndWriteFrame line LOG(INFO) << "Writing packet to stream."; gets run n times indicating the n packets were written to the stream. But ffprobe always shows only n - 1 frames int he video. And the final video doesn't play on quicktime.
What am I doing wrong??
Sorry for the delay but as i just had the same problem and noticed that this question deserves an answer, here how i solved this.
Up in front, the Problem only occured for me when using mov, mp4, 3gp as format. It worked frame accurate when using e.g. avi format. When i wrote uncompressed video frames to the container, i saw that the avi and mov had the same count of frames stored but the mov obviously had some problem in it's header.
Counting the number of frames in the mov using header metadata showed one frame is missing:
ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames -of default=nokey=1:noprint_wrappers=1 c:\temp\myinput.mov
While ignoring the index showed the correct number of frames:
-ignore_editlist 1
The solution for me was, set the timebase to the AVStream->CodeContext of the video stream.
The code above attempts to do this in this line:
out_stream_video->time_base = codec_context->time_base;
But the problem is that the posted code above does not expose the function GetCodecContext so we do not know if the time_base is correctly set for "codec_context". So it is my believe that the author's problem was that his function GetCodecContext did not set the time_base correctly.
I'm trying to stream video on a Raspberry Pi using the official V4L2 driver with the Raspberry Pi camera, from C++ on raspbian (2015-02 release), and I'm having low FPS issues.
Currently I'm just creating a window and copying the buffer to the screen (which takes about 30ms) whereas the select() takes about 140ms (for a total of 5-6 fps). I also tried sleeping for 100ms and it decreases the select() time by a similar amount (resulting in the same fps). CPU load is about 5-15%.
I also tried changing the driver fps from console (or system()) but it only works downwards (for example, if I set the driver fps to 1fps, I'll get 1fps but if I set it to 90fps I still get 5-6fps, even though the driver confirms setting it to 90fps).
Also, when querying FPS modes for the used resolution I get 90fps.
I included the parts of the code related to V4L2 (code omitted between different parts) :
//////////////////
// Open device
//////////////////
mFD = open(mDevName, O_RDWR | O_NONBLOCK, 0);
if (mFD == -1) ErrnoExit("Open device failed");
//////////////////
// Setup format
//////////////////
struct v4l2_format fmt;
memset(&fmt, 0, sizeof(fmt));
fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
Xioctl(VIDIOC_G_FMT, &fmt);
mImgWidth = fmt.fmt.pix.width;
mImgHeight = fmt.fmt.pix.height;
cout << "width=" << mImgWidth << " height=" << mImgHeight << "\nbytesperline=" << fmt.fmt.pix.bytesperline << " sizeimage=" << fmt.fmt.pix.sizeimage << "\n";
// For some reason querying the format always sets pixelformat to JPEG
// no matter the input, so set it back to YUYV
fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
if (Xioctl(VIDIOC_S_FMT, &fmt) == -1)
{
cout << "Set video format failed : " << strerror(errno) << "\n";
}
//////////////////
// Setup streaming
//////////////////
struct v4l2_requestbuffers req;
memset(&req, 0, sizeof(req));
req.count = 20;
req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
req.memory = V4L2_MEMORY_MMAP;
if (-1 == Xioctl(VIDIOC_REQBUFS, &req))
{
ErrnoExit("Reqbufs");
}
if (req.count < 2)
throw "Not enough buffer memory !";
mNBuffers = req.count;
mBuffers = new CBuffer[mNBuffers];
if (!mBuffers) throw "Out of memory !";
for (unsigned int i = 0; i < mNBuffers; i++)
{
struct v4l2_buffer buf;
memset(&buf, 0, sizeof(buf));
buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
buf.memory = V4L2_MEMORY_MMAP;
buf.index = i;
if (-1 == Xioctl(VIDIOC_QUERYBUF, &buf))
ErrnoExit("Querybuf");
mBuffers[i].mLength = buf.length;
mBuffers[i].pStart = mmap(NULL, buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, mFD, buf.m.offset);
if (mBuffers[i].pStart == MAP_FAILED)
ErrnoExit("mmap");
}
//////////////////
// Start streaming
//////////////////
unsigned int i;
enum v4l2_buf_type type;
struct v4l2_buffer buf;
for (i = 0; i < mNBuffers; i++)
{
memset(&buf, 0, sizeof(buf));
buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
buf.memory = V4L2_MEMORY_MMAP;
buf.index = i;
if (-1 == Xioctl(VIDIOC_QBUF, &buf))
ErrnoExit("QBUF");
}
type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
if (-1==Xioctl(VIDIOC_STREAMON, &type))
ErrnoExit("STREAMON");
And the last two parts in the main loop :
//////////////////
// Get frame
//////////////////
FD_ZERO(&fds);
FD_SET(mFD, &fds);
tv.tv_sec = 3;
tv.tv_usec = 0;
struct timespec t0, t1;
clock_gettime(CLOCK_REALTIME, &t0);
// This line takes about 140ms which I don't get
r = select(mFD + 1, &fds, NULL, NULL, &tv);
clock_gettime(CLOCK_REALTIME, &t1);
cout << "select time : " << ((float)(t1.tv_sec - t0.tv_sec))*1000.0f + ((float)(t1.tv_nsec - t0.tv_nsec))/1000000.0f << "\n";
if (-1 == r)
{
if (EINTR == errno)
continue;
ErrnoExit("select");
}
if (r == 0)
throw "Select timeout\n";
// Read the frame
//~ struct v4l2_buffer buf;
memset(&mCurBuf, 0, sizeof(mCurBuf));
mCurBuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
mCurBuf.memory = V4L2_MEMORY_MMAP;
// DQBUF about 2ms
if (-1 == Xioctl(VIDIOC_DQBUF, &mCurBuf))
{
if (errno == EAGAIN) continue;
ErrnoExit("DQBUF");
}
clock_gettime(CLOCK_REALTIME, &mCaptureTime);
// Manage frame in mBuffers[buf.index]
mCurBufIndex = mCurBuf.index;
break;
}
//////////////////
// Release frame
//////////////////
if (-1 == Xioctl(VIDIOC_QBUF, &mCurBuf))
ErrnoExit("VIDIOC_QBUF during mainloop");
I have been looking at the various methods of using the picamera and I'm hardly an expert, but it would seem that the default camera settings are what's holding you back. There are many modes and switches. I don't know if they are exposed through ioctls or how yet, I just started. But I had to use a program called v4l-ctl to get things ready for the mode I wanted. A deep look at that source and some code lifting should let you achieve greatness. Oh, and I doubt the select call is an issue, it's simply waiting on the descriptor which is slow to become readable. Depending on mode, etc. there can be a mandatory wait for autoexposure, etc.
Edit: I meant to say "a default setting" as you've changed some. There are also rules not codified in the driver.
The pixelformat matters. I encountered the similar low-fps problem, and I spent some time testing using my program in Go and C++ using V4L2 API. What I found is, Rpi Cam Module has good accelaration with H.264/MJPG pixelformat. I can easily get 60fps at 640*480, same as non-compressed formats like YUYV/RGB. However JPEG runs very slow. I can only get 4fps even at 320*240. And I also found the current is higher (>700mA) with JPEG compare to 500mA with H.264/MJPG.