ffmpeg/c++ Encode additional information of video frame with ffmpeg - c++

Question

I am new with ffmpeg & video encoding, after looking for some related questions on this page, I found this post which is very useful to understand the overview process of ffmpeg.
However, my work not only needs to manipulate with Mat frame, after extract important information from video (extract edge, position of edge block, type of each edge block, block number, motion vector), I have to encode and send them to client. I tried to find an example code for this part but it seems nobody have done it before.
My problems is how to encode these additional information along with video frame, and send both to client. I read about Huffman Coding which can help lossless compression, But is it possible encode edge & motion data using huffman coding while encoding video frame using ffmpeg? I'm doing experiment using udp protocol.
I can not find any information about this.
I read into metadata & side information in ffmpeg but it's not what I want to do.
I hope if you can give me an advice or a directions to research into this area, so I can understand and try to implement it. If there is any example code for this case, I would be very grateful for your sharing.
Thank you so much.
Below is encoder part on server side:
int encode(Mat& input_frame, EncodedCallback callback, void* userdata = nullptr) {
AVPacket pkt;
/* encode 1 second of video */
av_init_packet(&pkt);
pkt.data = NULL; // packet data will be allocated by the encoder
pkt.size = 0;
int size = 0;
fflush(stdout);
cvtFrame2AVFrameYUV420(input_frame, &frame);
static int time;
frame->pts = time++;
/* encode the image */
ret = avcodec_send_frame(c, frame);
if (ret < 0) {
fprintf(stderr, "Error avcodec_send_frame\n");
exit(1);
}
nbFramesEncoded++;
ret = avcodec_receive_packet(c, &pkt);
if (!isFirstFrameEmmited) {
nbNeededFramesInBuffer++;
printf("nbNeededFramesInBuffer: %d\n", nbNeededFramesInBuffer);
}
if (ret < 0) {
if (ret == -EAGAIN) {
//output is not available, we must send more input
} else {
fprintf(stderr, "Error avcodec_receive_packet %d\n", ret);
exit(1);
}
} else {
if (callback) {
callback(pkt, userdata);
}
size = pkt.size + 4;
av_packet_unref(&pkt);
}
return size;
}
Below is code to handle frame processing (presently we check & send motioned block to client)
void updateFrame(Mat& frame) {
//Get all Streams ready
bool isReady = true;
if (!frameStreamer->encoder->isFirstFrameEmmited) {
frameStreamer->sendFrame(frame);
isReady = false;
}
for (int yidx = 0; yidx < gridSize.height; yidx++) {
for (int xidx = 0; xidx < gridSize.width; xidx++) {
StreamPtr& stream = streamGrid[yidx][xidx];
if (!stream->encoder->isFirstFrameEmmited) {
Mat block = frame(stream->irect);
stream->sendFrame(block);
isReady = false;
}
}
}
if (isReady == false) {
return;
}
if (pGray.empty()) {
frameStreamer->sendFrame(frame);
frameStreamer->sendFrame(frame);
cvtColor(frame, pGray, CV_BGR2GRAY);
return;
}
//Motion Detection
Mat gray;
cvtColor(frame, gray, CV_BGR2GRAY);
Mat diff;
absdiff(gray, pGray, diff);
threshold(diff, diff, NOISE_THRESHOLD, 255, CV_THRESH_BINARY);
if (HEAT_IMAGE) {
gray.copyTo(diff, diff);
imshow("Gray", gray);
threshold(diff, diff, HEAT_THRESH, 255, CV_THRESH_TOZERO);
}
if (USE_MORPH_NOISE) {
Morph_Noise(diff);
}
Mat motionImg = Mat::zeros(frameSize, CV_8UC3);
//Block Classification
int nbModifiedBlocks = 0;
for (int yidx = 0; yidx < gridSize.height; yidx++) {
for (int xidx = 0; xidx < gridSize.width; xidx++) {
Rect irect(xidx * blockSize.width, yidx * blockSize.height,
blockSize.width, blockSize.height);
int blockDiff = sum(diff(irect))[0];
if (blockDiff > BLOCK_THRESHOLD * 255) {
this->blockCls.at<uchar>(yidx, xidx) = MODI_BLOCK;
nbModifiedBlocks++;
} else {
this->blockCls.at<uchar>(yidx, xidx) = SKIP_BLOCK;
}
}
}
//Send
if (nbModifiedBlocks > this->nbBlocksThresh) {
nbSentBytes += this->frameStreamer->sendFrame(frame);
} else {
for (int yidx = 0; yidx < gridSize.height; yidx++) {
for (int xidx = 0; xidx < gridSize.width; xidx++) {
uchar cls = this->blockCls.at<uchar>(yidx, xidx);
StreamPtr& stream = streamGrid[yidx][xidx];
bool send = false;
if (cls == MODI_BLOCK) {
if (DEBUG_NETWORK) {
printf("Normal (%d, %d): ", xidx, yidx);
}
send = true;
stream->encoder->nbFramesBuffered = stream->encoder->nbNeededFramesInBuffer;
rectangle(motionImg, stream->irect, Scalar(0, 0, 255), CV_FILLED);
} else if (stream->encoder->nbFramesBuffered > 0) {
if (DEBUG_NETWORK) {
printf("Extra (%d, %d): ", xidx, yidx);
}
send = true;
stream->encoder->nbFramesBuffered--;
stream->encoder->nbFlushFrames++;
rectangle(motionImg, stream->irect, Scalar(0, 255, 0), CV_FILLED);
}
if (send) {
Mat block = frame(stream->irect);
nbSentBytes += stream->sendFrame(block);
gray(stream->irect).copyTo(pGray(stream->irect));
}
}
}
}
}

Answer 1

I may not quite get what you ask, I think you asking adding additional stream to your output which may have some custom data. This is how you add data stream, add this after where you add video stream.
AVCodecParameters *par;
AVStream *st;
st = avformat_new_stream(ctx->oc, NULL); // ctx->oc is AVFormatContext *oc same you used where you add video stream
if (st == NULL)
{
fprintf(stderr, "Error: avformat_new_stream() failed.\n");
exit(EXIT_FAILURE);
}
par = st->codecpar;
par->codec_id = AV_CODEC_ID_TEXT; // raw UTF-8 text
par->codec_type = AVMEDIA_TYPE_DATA; // Opaque data information usually continuous
After that you can put your "data" to a pkt.data and send it with av_interleaved_write_frame after associated video frame output.