I trying to render a special image which is just a bunch of pixels with no format. It is a sort of a raw image with 42x 42 pixels. The images are palettes so I am curious how I should handle that situation too.
How can I convert an array of pixels to a texture in SDL2?
Do I apply palettes in the end?
More details:
Currently I am opening the image and adding some transparent (black bytes) and storing this into a char array. These pixels I will need to render.
#define _CRT_SECURE_NO_WARNINGS
#include <iostream>
#include <SDL.h>
#include <SDL_image.h>
#include <stdio.h>
#include <windows.h>
#include <vector>
#define SCREEN_WIDTH 640
#define SCREEN_HEIGHT 480
FILE* OpenCelFile(const char * filepath) {
FILE* ptr = fopen(filepath, "rb");
return ptr;
}
DWORD FrameCount[1] = { 0 };
DWORD DataStart[1] = { 0 };
DWORD dummy[1] = { 0 };
signed char CommandByte[1] = { 0 };
void SkipToData(FILE* pFile, DWORD* dataStart, int bytesRead)
{
int MoveForwardBytes = *dataStart - bytesRead;
fseek(pFile, MoveForwardBytes, SEEK_CUR);
}
// There is more to this , but it isn't 100% important.
void ReadCelHeader(FILE* pFile)
{
fread((void*)FrameCount, 4, 1, pFile);
fread((void*)DataStart, 4, 1, pFile);
SkipToData(pFile, DataStart, 8);
}
void ReadCommandByte(FILE* pFile)
{
fread((void*)CommandByte, 1, 1, pFile);
}
std::vector<char> backBuffer;
int CreateRawImageBuffer(FILE* pFile) {
ReadCommandByte(pFile);
int bytesRead = 0;
// handel transparent bytes;
if (*(BYTE*)CommandByte >= 0x80) {
// this is a negative byte
signed int skipBytes = *(BYTE*)CommandByte - 256;
// convert it to positive number.
skipBytes = abs(skipBytes);
for (skipBytes; skipBytes != NULL; skipBytes--) {
backBuffer.push_back(0x00);
bytesRead++;
}
}
// set real pixels
if (*(BYTE*)CommandByte < 0x80) {
signed int byteCount = *(BYTE*)CommandByte;
for (byteCount; byteCount != NULL; byteCount--) {
BYTE t_char[1] = {0x00};
fread((void*)t_char, 1, 1, pFile);
backBuffer.push_back(*t_char);
bytesRead++;
}
}
return bytesRead;
}
int main(int argc, char* args[]) {
SDL_Window* window = NULL;
SDL_Surface* screenSurface = NULL;
bool RunningMainGameLoop = true;
if (SDL_Init(SDL_INIT_VIDEO) < 0) {
fprintf(stderr, "could not initialize sdl2: %s\n", SDL_GetError());
return 1;
}
window = SDL_CreateWindow("Renderunkimage", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, SCREEN_WIDTH, SCREEN_HEIGHT, SDL_WINDOW_SHOWN );
if (window == NULL) {
fprintf(stderr, "could not create window: %s\n", SDL_GetError());
return 1;
}
FILE* ptr = OpenCelFile("C:\\Users\\luzer\\source\\repos\\unkimages\\unkimage.cel");
ReadCelHeader(ptr);
int TotalFramePixels = 48 * 48;
int fc = *FrameCount;
int pixelsAdded = { 0 };
for (fc; fc != 0; fc--) {
for (TotalFramePixels; TotalFramePixels != NULL; TotalFramePixels = TotalFramePixels - pixelsAdded) {
pixelsAdded = CreateRawImageBuffer(ptr);
}
}
screenSurface = SDL_GetWindowSurface(window);
SDL_FillRect(screenSurface, NULL, SDL_MapRGB(screenSurface->format, 0x00, 0x00, 0x00));
SDL_UpdateWindowSurface(window);
while (RunningMainGameLoop)
{
SDL_Event event;
while (SDL_PollEvent(&event)) {
switch (event.type) {
case SDL_QUIT:
RunningMainGameLoop = false;
break;
}
}
// handle renders...
}
SDL_DestroyWindow(window);
SDL_Quit();
return 0;
}
You need to use SDL_Surface first using the following:
SDL_CreateRGBSurfaceFrom(data, IMAGE_WIDTH, IMAGE_HEIGHT, 8, IMAGE_WIDTH /*pitch*/, 0, 0, 0, 0);
The images I am loading are really just single byte per colour and then I have to apply a palette to them. there is a sort of RLE encoding or compression which is why I have to load them , otherwise you just load them and pass them to a surface.
here is the whole code for anyone interested.
#define _CRT_SECURE_NO_WARNINGS
#include <iostream>
#include <SDL.h>
#include <SDL_image.h>
#include <stdio.h>
#include <windows.h>
#include <vector>
#include <fstream>
#include <iterator>
#include <algorithm>
#include <array>
#define SCREEN_WIDTH 200
#define SCREEN_HEIGHT 200
#define TICK_INTERVAL 45
static Uint32 next_time;
SDL_Color palData[256];
DWORD FrameCount[1] = { 0 };
DWORD DataStart[1] = { 0 };
DWORD dummy[1] = { 0 };
signed char CommandByte[1] = { 0 };
int TotalFramePixels = 48 * 48;
int IMAGE_WIDTH = 48;
int IMAGE_HEIGHT = 48;
struct Color {
uint8_t r;
uint8_t g;
uint8_t b;
};
struct celFrame {
byte ImageData[2304]; //fixed size of image. 48*48
};
FILE* OpenCelFile(const char * filepath) {
FILE* ptr = fopen(filepath, "rb");
return ptr;
}
void LoadPalette(const char* pszFileName)
{
FILE* PalFile = fopen(pszFileName, "rb");
for (int idx = 0; idx < 255; idx++) {
fread(&palData[idx].r, 1, 1, PalFile);
fread(&palData[idx].g, 1, 1, PalFile);
fread(&palData[idx].b, 1, 1, PalFile);
}
}
void SkipToData(FILE* pFile, DWORD* dataStart, int bytesRead)
{
int MoveForwardBytes = *dataStart - bytesRead;
fseek(pFile, MoveForwardBytes, SEEK_CUR);
}
// There is more to this , but it isn't 100% important.
void ReadCelHeader(FILE* pFile)
{
fread((void*)FrameCount, 4, 1, pFile);
fread((void*)DataStart, 4, 1, pFile);
SkipToData(pFile, DataStart, 8);
}
void ReadCommandByte(FILE* pFile)
{
fread((void*)CommandByte, 1, 1, pFile);
}
int CreateRawImageBuffer(FILE* pFile, struct celFrame* s_celFrame, int bytesRead) {
ReadCommandByte(pFile);
// handel transparent bytes;
if (*(BYTE*)CommandByte >= 0x80) {
// this is a negative byte
signed int skipBytes = *(BYTE*)CommandByte - 256;
// convert it to positive number.
skipBytes = abs(skipBytes);
for (skipBytes; skipBytes != NULL; skipBytes--) {
s_celFrame->ImageData[bytesRead] = 0xff;
bytesRead++;
}
}
// set real pixels
if (*(BYTE*)CommandByte < 0x80) {
signed int byteCount = *(BYTE*)CommandByte;
for (byteCount; byteCount != NULL; byteCount--) {
BYTE t_char[1] = {0xff};
fread((void*)t_char, 1, 1, pFile);
s_celFrame->ImageData[bytesRead] = *t_char;
bytesRead++;
}
}
if (bytesRead >= TotalFramePixels) {
return bytesRead;
}
CreateRawImageBuffer(pFile, s_celFrame, bytesRead);
}
Uint32 time_left(void)
{
Uint32 now;
now = SDL_GetTicks();
if (next_time <= now)
return 0;
else
return next_time - now;
}
int main(int argc, char* args[]) {
SDL_Window* window = NULL;
SDL_Surface* screenSurface = NULL;
SDL_Renderer* renderer = NULL;
SDL_Surface* PentagramSurface[8] = {0};
SDL_Texture* pentagramTexture[8] = { 0 };
int frameCount = 0;
bool RunningMainGameLoop = true;
if (SDL_Init(SDL_INIT_VIDEO) < 0) {
fprintf(stderr, "could not initialize sdl2: %s\n", SDL_GetError());
return 1;
}
window = SDL_CreateWindow("RenderCEL", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, SCREEN_WIDTH, SCREEN_HEIGHT, SDL_WINDOW_SHOWN );
if (window == nullptr) {
fprintf(stderr, "could not create window: %s\n", SDL_GetError());
return 1;
}
renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED);
if (renderer == nullptr) {
std::cout << "SDL_CreateRenderer Error: " << SDL_GetError() << std::endl;
return 1;
}
FILE* pfile = OpenCelFile("C:\\Users\\luzer\\source\\repos\\RenderCEL\\PentSpin.cel");
LoadPalette("C:\\Users\\luzer\\source\\luzer\\RenderCEL\\cut2.pal");
ReadCelHeader(pfile);
int fc = *FrameCount;
int pixelsAdded = { 0 };
celFrame s_celFrame[8] = {0};
// Create individual frames of each image
// by storing them in structs
for (int fidx = 0; fidx != fc; fidx++) {
int bytes_read = CreateRawImageBuffer(pfile, &s_celFrame[fidx], 0);
PentagramSurface[fidx] = SDL_CreateRGBSurfaceFrom(&s_celFrame[fidx], IMAGE_WIDTH, IMAGE_HEIGHT, 8, IMAGE_WIDTH /*pitch*/, 0, 0, 0, 0);
SDL_SetPaletteColors(PentagramSurface[fidx]->format->palette, palData, 0, 256);
pentagramTexture[fidx] = SDL_CreateTextureFromSurface(renderer, PentagramSurface[fidx]);
SDL_FreeSurface(PentagramSurface[fidx]);
}
screenSurface = SDL_GetWindowSurface(window);
SDL_FillRect(screenSurface, NULL, SDL_MapRGB(screenSurface->format, 0x00, 0x00, 0x00));
while (RunningMainGameLoop)
{
next_time = SDL_GetTicks() + TICK_INTERVAL;
SDL_Event event;
while (SDL_PollEvent(&event)) {
switch (event.type) {
case SDL_QUIT:
RunningMainGameLoop = false;
break;
}
}
// handle renders...
SDL_Rect drect = {0,0,48,48};
SDL_RenderClear(renderer);
SDL_RenderCopyEx(renderer, pentagramTexture[frameCount], NULL, NULL, 0, NULL, SDL_FLIP_VERTICAL);
SDL_RenderPresent(renderer);
SDL_Delay(time_left());
next_time += TICK_INTERVAL;
frameCount++;
if (frameCount == 8) { frameCount = 0; }
}
for (int idx = 0; idx < 8; idx++) {
SDL_DestroyTexture(pentagramTexture[idx]);
}
SDL_DestroyRenderer(renderer);
SDL_DestroyWindow(window);
SDL_Quit();
return 0;
}
Related
As part of a larger project I want to be able to capture and encode the desktop frame by frame in real time. I have the following test code to reproduce the issue shown in the screenshot:
#include <stdlib.h>
#include <stdio.h>
#include <iostream>
#include <fstream>
#include <string>
#include <string.h>
#include <math.h>
extern "C"
{
#include "libavdevice/avdevice.h"
#include "libavutil/channel_layout.h"
#include "libavutil/mathematics.h"
#include "libavutil/opt.h"
#include "libavformat/avformat.h"
#include "libswscale/swscale.h"
}
/* 5 seconds stream duration */
#define STREAM_DURATION 5.0
#define STREAM_FRAME_RATE 25 /* 25 images/s */
#define STREAM_NB_FRAMES ((int)(STREAM_DURATION * STREAM_FRAME_RATE))
#define STREAM_PIX_FMT AV_PIX_FMT_YUV420P /* default pix_fmt */
int videoStreamIndx;
int framerate = 30;
int width = 1920;
int height = 1080;
int encPacketCounter;
AVFormatContext* ifmtCtx;
AVCodecContext* avcodecContx;
AVFormatContext* ofmtCtx;
AVStream* videoStream;
AVCodecContext* avCntxOut;
AVPacket* avPkt;
AVFrame* avFrame;
AVFrame* outFrame;
SwsContext* swsCtx;
std::ofstream fs;
AVDictionary* ConfigureScreenCapture()
{
AVDictionary* options = NULL;
//Try adding "-rtbufsize 100M" as in https://stackoverflow.com/questions/6766333/capture-windows-screen-with-ffmpeg
av_dict_set(&options, "rtbufsize", "100M", 0);
av_dict_set(&options, "framerate", std::to_string(framerate).c_str(), 0);
char buffer[16];
sprintf(buffer, "%dx%d", width, height);
av_dict_set(&options, "video_size", buffer, 0);
return options;
}
AVCodecParameters* ConfigureAvCodec()
{
AVCodecParameters* av_codec_par_out = avcodec_parameters_alloc();
av_codec_par_out->width = width;
av_codec_par_out->height = height;
av_codec_par_out->bit_rate = 40000;
av_codec_par_out->codec_id = AV_CODEC_ID_H264; //AV_CODEC_ID_MPEG4; //Try H.264 instead of MPEG4
av_codec_par_out->codec_type = AVMEDIA_TYPE_VIDEO;
av_codec_par_out->format = 0;
return av_codec_par_out;
}
int GetVideoStreamIndex()
{
int VideoStreamIndx = -1;
avformat_find_stream_info(ifmtCtx, NULL);
/* find the first video stream index . Also there is an API available to do the below operations */
for (int i = 0; i < (int)ifmtCtx->nb_streams; i++) // find video stream position/index.
{
if (ifmtCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
{
VideoStreamIndx = i;
break;
}
}
if (VideoStreamIndx == -1)
{
}
return VideoStreamIndx;
}
void CreateFrames(AVCodecParameters* av_codec_par_in, AVCodecParameters* av_codec_par_out)
{
avFrame = av_frame_alloc();
avFrame->width = avcodecContx->width;
avFrame->height = avcodecContx->height;
avFrame->format = av_codec_par_in->format;
av_frame_get_buffer(avFrame, 0);
outFrame = av_frame_alloc();
outFrame->width = avCntxOut->width;
outFrame->height = avCntxOut->height;
outFrame->format = av_codec_par_out->format;
av_frame_get_buffer(outFrame, 0);
}
bool Init()
{
AVCodecParameters* avCodecParOut = ConfigureAvCodec();
AVDictionary* options = ConfigureScreenCapture();
AVInputFormat* ifmt = av_find_input_format("gdigrab");
auto ifmtCtxLocal = avformat_alloc_context();
if (avformat_open_input(&ifmtCtxLocal, "desktop", ifmt, &options) < 0)
{
return false;
}
ifmtCtx = ifmtCtxLocal;
videoStreamIndx = GetVideoStreamIndex();
AVCodecParameters* avCodecParIn = avcodec_parameters_alloc();
avCodecParIn = ifmtCtx->streams[videoStreamIndx]->codecpar;
AVCodec* avCodec = avcodec_find_decoder(avCodecParIn->codec_id);
if (avCodec == NULL)
{
return false;
}
avcodecContx = avcodec_alloc_context3(avCodec);
if (avcodec_parameters_to_context(avcodecContx, avCodecParIn) < 0)
{
return false;
}
//av_dict_set
int value = avcodec_open2(avcodecContx, avCodec, NULL); //Initialize the AVCodecContext to use the given AVCodec.
if (value < 0)
{
return false;
}
AVOutputFormat* ofmt = av_guess_format("h264", NULL, NULL);
if (ofmt == NULL)
{
return false;
}
auto ofmtCtxLocal = avformat_alloc_context();
avformat_alloc_output_context2(&ofmtCtxLocal, ofmt, NULL, NULL);
if (ofmtCtxLocal == NULL)
{
return false;
}
ofmtCtx = ofmtCtxLocal;
AVCodec* avCodecOut = avcodec_find_encoder(avCodecParOut->codec_id);
if (avCodecOut == NULL)
{
return false;
}
videoStream = avformat_new_stream(ofmtCtx, avCodecOut);
if (videoStream == NULL)
{
return false;
}
avCntxOut = avcodec_alloc_context3(avCodecOut);
if (avCntxOut == NULL)
{
return false;
}
if (avcodec_parameters_copy(videoStream->codecpar, avCodecParOut) < 0)
{
return false;
}
if (avcodec_parameters_to_context(avCntxOut, avCodecParOut) < 0)
{
return false;
}
avCntxOut->gop_size = 30; //3; //Use I-Frame frame every 30 frames.
avCntxOut->max_b_frames = 0;
avCntxOut->time_base.num = 1;
avCntxOut->time_base.den = framerate;
//avio_open(&ofmtCtx->pb, "", AVIO_FLAG_READ_WRITE);
if (avformat_write_header(ofmtCtx, NULL) < 0)
{
return false;
}
value = avcodec_open2(avCntxOut, avCodecOut, NULL); //Initialize the AVCodecContext to use the given AVCodec.
if (value < 0)
{
return false;
}
if (avcodecContx->codec_id == AV_CODEC_ID_H264)
{
av_opt_set(avCntxOut->priv_data, "preset", "ultrafast", 0);
av_opt_set(avCntxOut->priv_data, "zerolatency", "1", 0);
av_opt_set(avCntxOut->priv_data, "tune", "ull", 0);
}
if ((ofmtCtx->oformat->flags & AVFMT_GLOBALHEADER) != 0)
{
avCntxOut->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
CreateFrames(avCodecParIn, avCodecParOut);
swsCtx = sws_alloc_context();
if (sws_init_context(swsCtx, NULL, NULL) < 0)
{
return false;
}
swsCtx = sws_getContext(avcodecContx->width, avcodecContx->height, avcodecContx->pix_fmt,
avCntxOut->width, avCntxOut->height, avCntxOut->pix_fmt, SWS_FAST_BILINEAR,
NULL, NULL, NULL);
if (swsCtx == NULL)
{
return false;
}
return true;
}
void Encode(AVCodecContext* enc_ctx, AVFrame* frame, AVPacket* pkt)
{
int ret;
/* send the frame to the encoder */
ret = avcodec_send_frame(enc_ctx, frame);
if (ret < 0)
{
return;
}
while (ret >= 0)
{
ret = avcodec_receive_packet(enc_ctx, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return;
if (ret < 0)
{
return;
}
fs.write((char*)pkt->data, pkt->size);
av_packet_unref(pkt);
}
}
void EncodeFrames(int noFrames)
{
int frameCount = 0;
avPkt = av_packet_alloc();
AVPacket* outPacket = av_packet_alloc();
encPacketCounter = 0;
while (av_read_frame(ifmtCtx, avPkt) >= 0)
{
if (frameCount++ == noFrames)
break;
if (avPkt->stream_index != videoStreamIndx) continue;
avcodec_send_packet(avcodecContx, avPkt);
if (avcodec_receive_frame(avcodecContx, avFrame) >= 0) // Frame successfully decoded :)
{
outPacket->data = NULL; // packet data will be allocated by the encoder
outPacket->size = 0;
outPacket->pts = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
if (outPacket->dts != AV_NOPTS_VALUE)
outPacket->dts = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
outPacket->dts = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
outPacket->duration = av_rescale_q(1, avCntxOut->time_base, videoStream->time_base);
outFrame->pts = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
outFrame->pkt_duration = av_rescale_q(encPacketCounter, avCntxOut->time_base, videoStream->time_base);
encPacketCounter++;
int sts = sws_scale(swsCtx,
avFrame->data, avFrame->linesize, 0, avFrame->height,
outFrame->data, outFrame->linesize);
/* make sure the frame data is writable */
auto ret = av_frame_make_writable(outFrame);
if (ret < 0)
break;
Encode(avCntxOut, outFrame, outPacket);
}
av_frame_unref(avFrame);
av_packet_unref(avPkt);
}
}
void Dispose()
{
fs.close();
auto ifmtCtxLocal = ifmtCtx;
avformat_close_input(&ifmtCtx);
avformat_free_context(ifmtCtx);
avcodec_free_context(&avcodecContx);
}
int main(int argc, char** argv)
{
avdevice_register_all();
fs.open("out.h264");
if (Init())
{
EncodeFrames(300);
}
else
{
std::cout << "Failed to Init \n";
}
Dispose();
return 0;
}
As far as I can tell the setup of the encoding process is correct as it is largely unchanged from how the example given in the official documentation is working: https://libav.org/documentation/doxygen/master/encode__video_8c_source.html
However there is limited documentation around the desktop capture online so I am not sure if I have set that up correctly.
We have to open the out.h264 as binary file.
Replace fs.open("out.h264"); with fs.open("out.h264", std::ios::binary);.
The default file type in Windows is "text file".
That means that each \n in converted to \r\n when writing, and the encoded stream get "messed up".
It took me quite a long time to figure out the problem...
There is another small issue:
There is a missing loop at the end, that flushes the remaining encoded packets.
We can use FFprobe for counting the number of encoded frames:
ffprobe -v error -select_streams v:0 -count_frames -show_entries stream=nb_read_frames -print_format csv out.h264
The result is 263 instead of 300.
The solution is adding the following loop at the end of void EncodeFrames(int noFrames) function:
int ret = 0;
avcodec_send_frame(avCntxOut, NULL);
do
{
av_packet_unref(outPacket);
ret = avcodec_receive_packet(avCntxOut, outPacket);
if (!ret)
{
fs.write((char*)outPacket->data, outPacket->size);
}
} while (!ret);
This is not a solution to the problem directly but maybe so?
AVDictionary * pDic = NULL;
av_dict_set(&pDic, "tune", "zerolatency", 0);
avcodec_open2(avCntxOut, avCodecOut, &pDic);
After decoding some videos from iphone, I find the image has a strange border on the right.
It is not black but some random pix (width:4).
why i get the border?
how to cut this border or to turn it to black?
Most of videos dont have this problem.
here is the sample code:
the code will load a video from memory, and decode it
int VideoParseFFmpeg::read_packet(void *opaque, uint8_t *buf, int buf_size) {
struct buffer_data *bd = (struct buffer_data *) opaque;
buf_size = FFMIN(buf_size, bd->size_);
if (!buf_size)
return AVERROR_EOF;
memcpy(buf, bd->ptr_, buf_size);
bd->ptr_ += buf_size;
bd->size_ -= buf_size;
return buf_size;
}
int VideoParseFFmpeg::LoadContent(const std::string &video_content) {
int ret = 0;
...
video_size_ = video_content.size();
avio_ctx_buffer_size_ = video_size_ + AV_INPUT_BUFFER_PADDING_SIZE;
avio_ctx_buffer_ = static_cast<uint8_t *>(av_malloc(avio_ctx_buffer_size_));
bd_.ptr_ = (uint8_t *)(video_content.c_str());
bd_.size_ = video_content.size();
av_register_all();
input_ctx_ = avformat_alloc_context();
avio_ctx_ = avio_alloc_context(avio_ctx_buffer_,
avio_ctx_buffer_size_,
0,
&bd_,
&read_packet,
NULL,
NULL);
if ((ret = av_probe_input_buffer(avio_ctx_, &in_fmt_, "", NULL, 0, 0)) < 0) {
LOGGER_WARN(Log::GetLog(), "fail to prob input, err [{}]", AVERROR(ret));
return -1;
}
input_ctx_->pb = avio_ctx_;
/* open the input file */
if ((ret = avformat_open_input(&input_ctx_, "", in_fmt_, NULL)) != 0) {
LOGGER_WARN(Log::GetLog(), "fail to open input, err [{}]", AVERROR(ret));
return -1;
}
if ((ret = avformat_find_stream_info(input_ctx_, NULL)) < 0) {
LOGGER_WARN(Log::GetLog(),
"fail to find input stream information, err[{}]",
AVERROR(ret));
return -1;
}
/* find the video stream information */
if ((ret = av_find_best_stream(input_ctx_, AVMEDIA_TYPE_VIDEO, -1, -1, &decoder_, 0)) < 0) {
LOGGER_WARN(Log::GetLog(), "fail to find a video stream from input, err[{}]", ret);
return -1;
}
video_stream_idx_ = ret;
if (!(decoder_ctx_ = avcodec_alloc_context3(decoder_))) {
LOGGER_WARN(Log::GetLog(), "fail to alloc avcodec context");
return -1;
}
video_stream_ = input_ctx_->streams[video_stream_idx_];
if ((ret = avcodec_parameters_to_context(decoder_ctx_, video_stream_->codecpar)) < 0) {
LOGGER_WARN(Log::GetLog(), "fail to convert parameters to context, err [{}]", ret);
return -1;
}
if ((ret = avcodec_open2(decoder_ctx_, decoder_, NULL)) < 0) {
LOGGER_WARN(Log::GetLog(), "fail to open decodec, err[{}]", ret);
return -1;
}
return 0;
}
void VideoParseFFmpeg::GetFrames(std::vector<cv::Mat> &frames) {
while (1) {
...
decode_write(decoder_ctx_, &packet_, &buffer, frames);
...
}
//flush
...
}
cv::Mat VideoParseFFmpeg::avFrame2Mat(AVFrame *pAvFrame,
AVCodecContext *pCodecCtx,
AVPixelFormat src_fmt) {
AVFrame *pFrameBGR = av_frame_alloc(); // 存储解码后转换的BGR数据
int w = pAvFrame->width;
int h = pAvFrame->height;
auto size = av_image_get_buffer_size(AV_PIX_FMT_BGR24, w, h, 1);
cv::Mat out(h, w, CV_8UC3);
int ret = av_image_fill_arrays(pFrameBGR->data,
pFrameBGR->linesize,
out.data,
AV_PIX_FMT_BGR24,
w,
h,
1);
img_convert_ctx_ = sws_getCachedContext(img_convert_ctx_,
w,
h,
ConvertDeprecatedFormat(src_fmt),
w,
h,
AV_PIX_FMT_BGR24,
SWS_BICUBIC,
NULL, NULL, NULL);
sws_scale(img_convert_ctx_,
(const uint8_t *const *) pAvFrame->data,
pAvFrame->linesize,
0,
pCodecCtx->height,
pFrameBGR->data,
pFrameBGR->linesize);
av_free(pFrameBGR);
return out;
}
int VideoParseFFmpeg::decode_write(AVCodecContext *avctx,
AVPacket *packet,
uint8_t **buffer,
std::vector<cv::Mat> &p_mat_out) {
AVFrame *frame = NULL, *sw_frame = NULL;
AVFrame *tmp_frame = NULL;
int ret = 0;
ret = avcodec_send_packet(avctx, packet);
...
while (1) {
auto clear = [&frame, &sw_frame, this] {
if (frame != NULL)
av_frame_free(&frame);
if (sw_frame != NULL)
av_frame_free(&sw_frame);
av_packet_unref(&packet_);
};
...
ret = avcodec_receive_frame(avctx, frame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
clear();
return 0;
} else if (ret < 0) {
LOGGER_WARN(Log::GetLog(), "error while decoding, err[{}]", AVERROR(ret));
clear();
return ret;
}
tmp_frame = frame;
p_mat_out.push_back(avFrame2Mat(tmp_frame,
avctx,
(AVPixelFormat) tmp_frame->format));
clear();
}
}
After replacing lib tcmalloc with a newer version, the border turns to be a black line (used to be a random line), like this.
there is image content in the area of the border in the original image, how to fix it?
So, I picked up SDL and started writing a simple game. There is some problem I'm very confused about when I try to draw two textures to the screen using SDL_RenderCopy. The important part is inside the while(!quit) where there are two SDL_RenderCopy calls. For some reason only the second one draws something on the screen. Here's my code:
#include <SDL.h>
#include <SDL_image.h>
#include <stdio.h>
const int SCREEN_WIDTH = 1280;
const int SCREEN_HEIGHT = 720;
enum TEXTURES{
T_GRASS,
T_ALL
};
bool init();
bool loadMedia(const int texture, char* path);
void close();
SDL_Texture* loadTexture(char* path);
SDL_Window* window = NULL;
SDL_Renderer* renderer = NULL;
SDL_Texture* textures[] = {NULL};
int main(int argc, char* args[]){
if(!init())
printf("Failed to initialize!\n");
else
if(!loadMedia(T_GRASS, "images/tGrass.png"))
printf("Failed to load media!\n");
else{
bool quit = false;
SDL_Event e;
while(!quit){
while(SDL_PollEvent(&e) != 0)
if(e.type == SDL_QUIT)
quit = true;
SDL_RenderClear(renderer);
SDL_Rect pos;
pos.h = 16;
pos.w = 16;
pos.x = 0;
pos.y = 0;
SDL_Rect pos1;
pos1.h = 16;
pos1.w = 16;
pos.x = 16;
pos.y = 16;
SDL_RenderCopy(renderer,textures[T_GRASS],NULL,&pos);
SDL_RenderCopy(renderer,textures[T_GRASS],NULL,&pos1);
SDL_RenderPresent(renderer);
}
}
close();
return 0;
}
bool init(){
bool success = true;
if(SDL_Init(SDL_INIT_VIDEO) < 0){
printf("SDL could not initialize! Error: %s\n", SDL_GetError());
success = false;
}else{
if(!SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "1"))
printf("WARNING: Linear texture filtering not enabled!\n");
window = SDL_CreateWindow("openZ", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, SCREEN_WIDTH, SCREEN_HEIGHT, SDL_WINDOW_SHOWN);
if(window == NULL){
printf("Window could not be created! Error: %s\n", SDL_GetError());
success = false;
}else{
renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED);
if(renderer == NULL){
printf("Render could not be created! Error: %s\n", SDL_GetError());
success = false;
}else{
SDL_SetRenderDrawColor(renderer, 0xFF, 0xFF, 0xFF, 0xFF);
int imgFlags = IMG_INIT_PNG;
if(!(IMG_Init(imgFlags) & imgFlags)){
printf("SDL Image could not initialize! Error: %s\n", SDL_GetError());
success = false;
}
}
}
}
return success;
}
bool loadMedia(const int texture, char* path){
bool success = true;
textures[texture] = loadTexture(path);
if(textures[texture] == NULL){
printf("Failed to load texture image %s!\n", path);
success = false;
}
return success;
}
void close(){
for(int i = 0; i < T_ALL; i++){
SDL_DestroyTexture(textures[i]);
textures[i] = NULL;
}
SDL_DestroyRenderer(renderer);
SDL_DestroyWindow(window);
window = NULL;
renderer = NULL;
IMG_Quit();
SDL_Quit();
}
SDL_Texture* loadTexture(char* path){
SDL_Texture* newTexture = NULL;
SDL_Surface* loadedSurface = IMG_Load(path);
if(loadedSurface == NULL){
printf("Unable to load image %s! Error: %s\n", path, IMG_GetError());
}else{
newTexture = SDL_CreateTextureFromSurface(renderer, loadedSurface);
if(newTexture == NULL)
printf("Unable to create texture from %s! Error: %s\n", path, SDL_GetError());
SDL_FreeSurface(loadedSurface);
}
return newTexture;
}
Sorry! My bad.. turns out I had 2 typos which made the two textures render over each other. I wrote:
SDL_Rect pos;
pos.h = 16;
pos.w = 16;
pos.x = 0;
pos.y = 0;
SDL_Rect pos1;
pos1.h = 16;
pos1.w = 16;
pos.x = 16;
pos.y = 16;
where it should have been:
SDL_Rect pos;
pos.h = 16;
pos.w = 16;
pos.x = 0;
pos.y = 0;
SDL_Rect pos1;
pos1.h = 16;
pos1.w = 16;
pos1.x = 16;
pos1.y = 16;
^^ the last too lines were referencing the wrong rectangle.
I am having issues encoding screen captures, into a h.264 file for viewing. The program below is cobbled together from examples here and here. The first example, uses an older version of the ffmpeg api. So I tried to update that example for use in my program. The file is created and has something written to it, but when I view the file. The encoded images are all distorted. I am able to run the video encoding example from the ffmpeg api successfully. This is my first time posting, so if I missed anything please let me know.
I appreciate any assistance that is given.
My program:
#include <Windows.h>
#include <string>
#include <sstream>
#include <time.h>
#include <iostream>
#include <dwmapi.h>
extern "C"{
#include <stdint.h>
#include <libavcodec/avcodec.h>
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
#include <libavutil/opt.h>
}
using namespace std;
void ScreenShot(const char* BmpName, uint8_t *frame)
{
HWND DesktopHwnd = GetDesktopWindow();
RECT DesktopParams;
HDC DevC = GetDC(DesktopHwnd);
GetWindowRect(DesktopHwnd,&DesktopParams);
DWORD Width = DesktopParams.right - DesktopParams.left;
DWORD Height = DesktopParams.bottom - DesktopParams.top;
DWORD FileSize = sizeof(BITMAPFILEHEADER)+sizeof(BITMAPINFOHEADER)+(sizeof(RGBTRIPLE)+1*(Width*Height*4));
char *BmpFileData = (char*)GlobalAlloc(0x0040,FileSize);
PBITMAPFILEHEADER BFileHeader = (PBITMAPFILEHEADER)BmpFileData;
PBITMAPINFOHEADER BInfoHeader = (PBITMAPINFOHEADER)&BmpFileData[sizeof(BITMAPFILEHEADER)];
BFileHeader->bfType = 0x4D42; // BM
BFileHeader->bfSize = sizeof(BITMAPFILEHEADER);
BFileHeader->bfOffBits = sizeof(BITMAPFILEHEADER)+sizeof(BITMAPINFOHEADER);
BInfoHeader->biSize = sizeof(BITMAPINFOHEADER);
BInfoHeader->biPlanes = 1;
BInfoHeader->biBitCount = 32;
BInfoHeader->biCompression = BI_RGB;
BInfoHeader->biHeight = Height;
BInfoHeader->biWidth = Width;
RGBTRIPLE *Image = (RGBTRIPLE*)&BmpFileData[sizeof(BITMAPFILEHEADER)+sizeof(BITMAPINFOHEADER)];
RGBTRIPLE color;
//pPixels = (RGBQUAD **)new RGBQUAD[sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER)];
int start = clock();
HDC CaptureDC = CreateCompatibleDC(DevC);
HBITMAP CaptureBitmap = CreateCompatibleBitmap(DevC,Width,Height);
SelectObject(CaptureDC,CaptureBitmap);
BitBlt(CaptureDC,0,0,Width,Height,DevC,0,0,SRCCOPY|CAPTUREBLT);
GetDIBits(CaptureDC,CaptureBitmap,0,Height,frame,(LPBITMAPINFO)BInfoHeader, DIB_RGB_COLORS);
int end = clock();
cout << "it took " << end - start << " to capture a frame" << endl;
DWORD Junk;
HANDLE FH = CreateFileA(BmpName,GENERIC_WRITE,FILE_SHARE_WRITE,0,CREATE_ALWAYS,0,0);
WriteFile(FH,BmpFileData,FileSize,&Junk,0);
CloseHandle(FH);
GlobalFree(BmpFileData);
}
void video_encode_example(const char *filename, int codec_id)
{
AVCodec *codec;
AVCodecContext *c= NULL;
int i, ret, x, y, got_output;
FILE *f;
AVFrame *frame;
AVPacket pkt;
uint8_t endcode[] = { 0, 0, 1, 0xb7 };
printf("Encode video file %s\n", filename);
/* find the mpeg1 video encoder */
codec = avcodec_find_encoder(AV_CODEC_ID_H264);
if (!codec) {
fprintf(stderr, "Codec not found\n");
cin.get();
exit(1);
}
c = avcodec_alloc_context3(codec);
if (!c) {
fprintf(stderr, "Could not allocate video codec context\n");
cin.get();
exit(1);
}
/* put sample parameters */
c->bit_rate = 400000;
/* resolution must be a multiple of two */
c->width = 352;
c->height = 288;
/* frames per second */
c->time_base.num=1;
c->time_base.den = 25;
c->gop_size = 10; /* emit one intra frame every ten frames */
c->max_b_frames=1;
c->pix_fmt = AV_PIX_FMT_YUV420P;
if(codec_id == AV_CODEC_ID_H264)
av_opt_set(c->priv_data, "preset", "slow", 0);
/* open it */
if (avcodec_open2(c, codec, NULL) < 0) {
fprintf(stderr, "Could not open codec\n");
exit(1);
}
f = fopen(filename, "wb");
if (!f) {
fprintf(stderr, "Could not open %s\n", filename);
exit(1);
}
frame = av_frame_alloc();
if (!frame) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
frame->format = c->pix_fmt;
frame->width = c->width;
frame->height = c->height;
/* the image can be allocated by any means and av_image_alloc() is
just the most convenient way if av_malloc() is to be used */
ret = av_image_alloc(frame->data, frame->linesize, c->width, c->height, c->pix_fmt, 32);
if (ret < 0) {
fprintf(stderr, "Could not allocate raw picture buffer\n");
exit(1);
}
/* encode 1 second of video */
for(i=0;i<250;i++) {
av_init_packet(&pkt);
pkt.data = NULL; // packet data will be allocated by the encoder
pkt.size = 0;
fflush(stdout);
/* prepare a dummy image */
/* Y */
for(y=0;y<c->height;y++) {
for(x=0;x<c->width;x++) {
frame->data[0][y * frame->linesize[0] + x] = x + y + i * 3;
}
}
/* Cb and Cr */
for(y=0;y<c->height/2;y++) {
for(x=0;x<c->width/2;x++) {
frame->data[1][y * frame->linesize[1] + x] = 128 + y + i * 2;
frame->data[2][y * frame->linesize[2] + x] = 64 + x + i * 5;
}
}
frame->pts = i;
/* encode the image */
ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
if (ret < 0) {
fprintf(stderr, "Error encoding frame\n");
exit(1);
}
if (got_output) {
printf("Write frame %3d (size=%5d)\n", i, pkt.size);
fwrite(pkt.data, 1, pkt.size, f);
av_free_packet(&pkt);
}
}
/* get the delayed frames */
for (got_output = 1; got_output; i++) {
fflush(stdout);
ret = avcodec_encode_video2(c, &pkt, NULL, &got_output);
if (ret < 0) {
fprintf(stderr, "Error encoding frame\n");
exit(1);
}
if (got_output) {
printf("Write frame %3d (size=%5d)\n", i, pkt.size);
fwrite(pkt.data, 1, pkt.size, f);
av_free_packet(&pkt);
}
}
/* add sequence end code to have a real mpeg file */
fwrite(endcode, 1, sizeof(endcode), f);
fclose(f);
avcodec_close(c);
av_free(c);
av_freep(&frame->data[0]);
av_frame_free(&frame);
printf("\n");
}
void write_video_frame()
{
}
int lineSizeOfFrame(int width)
{
return (width*24 + 31)/32 * 4;//((width*24 / 8) + 3) & ~3;//(width*24 + 31)/32 * 4;
}
int getScreenshotWithCursor(uint8_t* frame)
{
int successful = 0;
HDC screen, bitmapDC;
HBITMAP screen_bitmap;
screen = GetDC(NULL);
RECT DesktopParams;
HWND desktop = GetDesktopWindow();
GetWindowRect(desktop, &DesktopParams);
int width = DesktopParams.right;
int height = DesktopParams.bottom;
bitmapDC = CreateCompatibleDC(screen);
screen_bitmap = CreateCompatibleBitmap(screen, width, height);
SelectObject(bitmapDC, screen_bitmap);
if (BitBlt(bitmapDC, 0, 0, width, height, screen, 0, 0, SRCCOPY))
{
int pos_x, pos_y;
HICON hcur;
ICONINFO icon_info;
CURSORINFO cursor_info;
cursor_info.cbSize = sizeof(CURSORINFO);
if (GetCursorInfo(&cursor_info))
{
if (cursor_info.flags == CURSOR_SHOWING)
{
hcur = CopyIcon(cursor_info.hCursor);
if (GetIconInfo(hcur, &icon_info))
{
pos_x = cursor_info.ptScreenPos.x - icon_info.xHotspot;
pos_y = cursor_info.ptScreenPos.y - icon_info.yHotspot;
DrawIcon(bitmapDC, pos_x, pos_y, hcur);
if (icon_info.hbmColor) DeleteObject(icon_info.hbmColor);
if (icon_info.hbmMask) DeleteObject(icon_info.hbmMask);
}
}
}
int header_size = sizeof(BITMAPINFOHEADER) + 256*sizeof(RGBQUAD);
size_t line_size = lineSizeOfFrame(width);
PBITMAPINFO lpbi = (PBITMAPINFO) malloc(header_size);
lpbi->bmiHeader.biSize = header_size;
lpbi->bmiHeader.biWidth = width;
lpbi->bmiHeader.biHeight = height;
lpbi->bmiHeader.biPlanes = 1;
lpbi->bmiHeader.biBitCount = 24;
lpbi->bmiHeader.biCompression = BI_RGB;
lpbi->bmiHeader.biSizeImage = height*line_size;
lpbi->bmiHeader.biXPelsPerMeter = 0;
lpbi->bmiHeader.biYPelsPerMeter = 0;
lpbi->bmiHeader.biClrUsed = 0;
lpbi->bmiHeader.biClrImportant = 0;
if (GetDIBits(bitmapDC, screen_bitmap, 0, height, (LPVOID)frame, lpbi, DIB_RGB_COLORS))
{
int i;
uint8_t *buf_begin = frame;
uint8_t *buf_end = frame + line_size*(lpbi->bmiHeader.biHeight - 1);
void *temp = malloc(line_size);
for (i = 0; i < lpbi->bmiHeader.biHeight / 2; ++i)
{
memcpy(temp, buf_begin, line_size);
memcpy(buf_begin, buf_end, line_size);
memcpy(buf_end, temp, line_size);
buf_begin += line_size;
buf_end -= line_size;
}
cout << *buf_begin << endl;
free(temp);
successful = 1;
}
free(lpbi);
}
DeleteObject(screen_bitmap);
DeleteDC(bitmapDC);
ReleaseDC(NULL, screen);
return successful;
}
int main()
{
RECT DesktopParams;
HWND desktop = GetDesktopWindow();
GetWindowRect(desktop, &DesktopParams);
int width = DesktopParams.right;
int height = DesktopParams.bottom;
uint8_t *frame = (uint8_t *)malloc(width * height);
AVCodec *codec;
AVCodecContext *codecContext = NULL;
AVPacket packet;
FILE *f;
AVFrame *pictureYUV = NULL;
AVFrame *pictureRGB;
avcodec_register_all();
codec = avcodec_find_encoder(AV_CODEC_ID_H264);
if(!codec)
{
cout << "codec not found!" << endl;
cin.get();
return 1;
}
else
{
cout << "codec h265 found!" << endl;
}
codecContext = avcodec_alloc_context3(codec);
codecContext->bit_rate = width * height * 4;
codecContext->width = width;
codecContext->height = height;
codecContext->time_base.num = 1;
codecContext->time_base.den = 250;
codecContext->gop_size = 10;
codecContext->max_b_frames = 1;
codecContext->keyint_min = 1;
codecContext->i_quant_factor = (float)0.71; // qscale factor between P and I frames
codecContext->b_frame_strategy = 20; ///// find out exactly what this does
codecContext->qcompress = (float)0.6; ///// find out exactly what this does
codecContext->qmin = 20; // minimum quantizer
codecContext->qmax = 51; // maximum quantizer
codecContext->max_qdiff = 4; // maximum quantizer difference between frames
codecContext->refs = 4; // number of reference frames
codecContext->trellis = 1;
codecContext->pix_fmt = AV_PIX_FMT_YUV420P;
codecContext->codec_id = AV_CODEC_ID_H264;
codecContext->codec_type = AVMEDIA_TYPE_VIDEO;
if(avcodec_open2(codecContext, codec, NULL) < 0)
{
cout << "couldn't open codec" << endl;
cout << stderr << endl;
cin.get();
return 1;
}
else
{
cout << "opened h265 codec!" << endl;
cin.get();
}
f = fopen("test.h264", "wb");
if(!f)
{
cout << "Unable to open file" << endl;
return 1;
}
struct SwsContext *img_convert_ctx = sws_getContext(codecContext->width, codecContext->height, PIX_FMT_RGB32, codecContext->width,
codecContext->height, codecContext->pix_fmt, SWS_BILINEAR, NULL, NULL, NULL);
int got_output = 0, i = 0;
uint8_t encode[] = { 0, 0, 1, 0xb7 };
try
{
for(i = 0; i < codecContext->time_base.den; i++)
{
av_init_packet(&packet);
packet.data = NULL;
packet.size = 0;
pictureRGB = av_frame_alloc();
pictureYUV = av_frame_alloc();
getScreenshotWithCursor(frame);
//ScreenShot("example.bmp", frame);
int nbytes = avpicture_get_size(AV_PIX_FMT_YUV420P, codecContext->width, codecContext->height); // allocating outbuffer
uint8_t* outbuffer = (uint8_t*)av_malloc(nbytes*sizeof(uint8_t));
pictureRGB = av_frame_alloc();
pictureYUV = av_frame_alloc();
avpicture_fill((AVPicture*)pictureRGB, frame, PIX_FMT_RGB32, codecContext->width, codecContext->height); // fill image with input screenshot
avpicture_fill((AVPicture*)pictureYUV, outbuffer, PIX_FMT_YUV420P, codecContext->width, codecContext->height);
av_image_alloc(pictureYUV->data, pictureYUV->linesize, codecContext->width, codecContext->height, codecContext->pix_fmt, 32);
sws_scale(img_convert_ctx, pictureRGB->data, pictureRGB->linesize, 0, codecContext->height, pictureYUV->data, pictureYUV->linesize);
pictureYUV->pts = i;
avcodec_encode_video2(codecContext, &packet, pictureYUV, &got_output);
if(got_output)
{
printf("Write frame %3d (size=%5d)\n", i, packet.size);
fwrite(packet.data, 1, packet.size, f);
av_free_packet(&packet);
}
//av_frame_free(&pictureRGB);
//av_frame_free(&pictureYUV);
}
for(got_output = 1; got_output; i++)
{
fflush(stdout);
avcodec_encode_video2(codecContext, &packet, NULL, &got_output);
if (got_output) {
printf("Write frame %3d (size=%5d)\n", i, packet.size);
fwrite(packet.data, 1, packet.size, f);
av_free_packet(&packet);
}
}
}
catch(std::exception ex)
{
cout << ex.what() << endl;
}
avcodec_close(codecContext);
av_free(codecContext);
av_freep(&pictureYUV->data[0]);
//av_frame_free(&picture);
fwrite(encode, 1, sizeof(encode), f);
fclose(f);
cin.get();
return 0;
}
I got distorted image when try to convert YUV420p to RGB24 using
sws_scale.
Code:
ret = avcodec_decode_video2(video_dec_ctx, frame, got_frame, &pkt);
if (ret < 0) {
fprintf(stderr, "Error decoding video frame\n");
return ret;
}
if (*got_frame)
{
printf("video_frame%s n:%d coded_n:%d pts:%s\n",
cached ? "(cached)" : "",
video_frame_count++, frame->coded_picture_number,
"#"/*av_ts2timestr(frame->pts, &video_dec_ctx->time_base)*/);
/* copy decoded frame to destination buffer:
* this is required since rawvideo expects non aligned data */
av_image_copy(video_dst_data, video_dst_linesize,
(const uint8_t **)(frame->data), frame->linesize,
video_dec_ctx->pix_fmt, video_dec_ctx->width, video_dec_ctx->height);
/* write to rawvideo file */
fwrite(video_dst_data[0], 1, video_dst_bufsize, video_dst_file);
AVPicture pic;
avpicture_alloc( &pic, AV_PIX_FMT_RGB24, frame->width, frame->height);
SwsContext *ctxt = sws_getContext(frame->width, frame->height, static_cast<AVPixelFormat>(frame->format),
frame->width, frame->height, AV_PIX_FMT_RGB24, SWS_BILINEAR, NULL, NULL, NULL);
if ( NULL == ctxt )
{
//Log("failed to get sws context");
}
if ( 0 < sws_scale(ctxt, frame->data, frame->linesize, 0, frame->height, pic.data, pic.linesize))
{
char szPic[256] = { 0 };
sprintf( szPic, "decoded/%d.bmp", video_frame_count );
FILE *pf = fopen(szPic,"w");
if ( NULL != pf )
{
BITMAPFILEHEADER bmpFileHeader = {0};
bmpFileHeader.bfReserved1 = 0;
bmpFileHeader.bfReserved2 = 0;
bmpFileHeader.bfType = 0x4D42;
bmpFileHeader.bfSize = sizeof(bmpFileHeader) + sizeof(BITMAPINFOHEADER) + pic.linesize[0] * frame->height;
bmpFileHeader.bfOffBits = sizeof(bmpFileHeader) + sizeof(BITMAPINFOHEADER);
BITMAPINFOHEADER bmiHeader = { 0 };
bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
bmiHeader.biWidth = frame->width;
bmiHeader.biHeight = 0 - frame->height;
bmiHeader.biPlanes = 1;
bmiHeader.biBitCount = 24;
bmiHeader.biCompression = BI_RGB;
bmiHeader.biSizeImage = pic.linesize[0] * frame->height;
bmiHeader.biXPelsPerMeter = 0;
bmiHeader.biYPelsPerMeter = 0;
bmiHeader.biClrUsed = 0;
bmiHeader.biClrImportant = 0;
fwrite( &bmpFileHeader, 1, sizeof(bmpFileHeader), pf );
fwrite( &bmiHeader, 1, sizeof(bmiHeader), pf );
fwrite( pic.data[0], 1, pic.linesize[0] * frame->height, pf );
fclose( pf );
}
}
// pic.data[0] now contains the image data in RGB format (3 bytes)
// and pic.linesize[0] is the pitch of the data (ie. size of a row in memory, which can be larger than width*sizeof(pixel))
avpicture_free(&pic);
sws_freeContext(ctxt);
}
above only decode frame then convert this from to RGB24, then write a bitmap.
original video frame like this,
but converted image,
is there missing some code or some code is wrong?
thanks in advance.
fwrite( pic.data[0], 1, pic.linesize[0] * frame->height, pf );
For an image of e.g. 1280x720, linesize is typically larger, e.g. 1312, so you'll be writing more data than image size if you write linesize*height. You want to write (in a loop) width pixels offset by linesize bytes:
uint8_t *ptr = pic.data[0];
for (int y = 0; y < frame->height; y++) {
fwrite(ptr, 1, frame->width, pf);
ptr += pic.linesize[0];
}
And then it should work correctly.
maybe these codes can help you. these works good.
int got_frame = 0;
auto len = avcodec_decode_video2(m_avCodecContext
, m_avFrame
, &got_frame
, &avpkt);
if (len < 0)
{
return;
}
if (got_frame /*&& !silentMode*/)
{
//if (videoRenderer != nullptr)
{
if (frameSize == NULL)
{
return;
}
uint8_t *dst_data[4];
int dst_linesize[4];
int dst_w, dst_h;
int ret = 0;
if (1)// avcodec_alloc_frame()
{
auto stride = m_avFrame->linesize;
auto scan0 = m_avFrame->data;
SwsContext *scaleContext = sws_getContext(m_avCodecContext->width
, m_avCodecContext->height
, m_avCodecContext->pix_fmt
, m_avCodecContext->width
, m_avCodecContext->height
, PixelFormat::PIX_FMT_BGR24
, SWS_FAST_BILINEAR, NULL, NULL, NULL);
if (scaleContext == NULL)
{
//TODO: log error
return;
}
try
{
//*vb->signal = 1;
ret = avpicture_alloc(&m_dst_picture
, PixelFormat::PIX_FMT_BGR24
, m_avCodecContext->width
, m_avCodecContext->height);
// AVFrame *picture_RGB;
// uint8_t *bufferRGB;
// picture_RGB = avcodec_alloc_frame();
// bufferRGB = (uint8_t*)malloc(720*576*(24/8)/*avpicture_get_size(PIX_FMT_RGB24, 720, 576)*/);
// avpicture_fill((AVPicture *)picture_RGB, bufferRGB, PIX_FMT_RGB24, 720, 576);
if (ret < 0)
{
return;
}
int retScale = sws_scale(scaleContext
, scan0
, stride
, 0
, m_avCodecContext->height
, m_dst_picture.data //picture_RGB->data
, m_dst_picture.linesize //picture_RGB->linesize
);
if (1)
{
HWND hwnd = m_pParent->GetSafeHwnd();
SetFocus(hwnd);
CRect rc;
m_pParent->GetClientRect(rc);
CDC *cdc = m_pParent->GetDC();
char* bitmap = (char*)m_dst_picture.data[0];
// static unsigned int i = 0;
// bmp_save(bitmap, m_avCodecContext->width, m_avCodecContext->height, i++);
BITMAPINFO bmpinfo;
bmpinfo.bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
bmpinfo.bmiHeader.biWidth = m_avCodecContext->width;
bmpinfo.bmiHeader.biHeight = -m_avCodecContext->height;
bmpinfo.bmiHeader.biPlanes = 1;
bmpinfo.bmiHeader.biBitCount = 24;
bmpinfo.bmiHeader.biCompression = BI_RGB;
bmpinfo.bmiHeader.biSizeImage =
m_avCodecContext->width * m_avCodecContext->height * (24 / 8);
bmpinfo.bmiHeader.biXPelsPerMeter = 100;
bmpinfo.bmiHeader.biYPelsPerMeter = 100;
bmpinfo.bmiHeader.biClrUsed = 0;
bmpinfo.bmiHeader.biClrImportant = 0;
HBITMAP hBitmap = CreateDIBitmap(cdc->GetSafeHdc(), &bmpinfo.bmiHeader, CBM_INIT, bitmap, &bmpinfo/*bi*/, DIB_RGB_COLORS);
DrawBitmap(cdc, hBitmap, m_pParent);
::DeleteObject(hBitmap);
::DeleteObject(cdc->GetSafeHdc());
}
avpicture_free(&m_dst_picture);
sws_freeContext(scaleContext);
}
catch (int e)
{
sws_freeContext(scaleContext);
}
}
}
}
void DrawBitmap(CDC *pDC, HBITMAP hbitmap,CWnd *wnd)
{
CBitmap *pBitmap = CBitmap::FromHandle(hbitmap);
BITMAP bm;
pBitmap -> GetBitmap(&bm);
CDC MemDC;
MemDC.CreateCompatibleDC(pDC);
HGDIOBJ gob= MemDC.SelectObject(pBitmap);
CRect rc;
wnd->GetClientRect(rc);
pDC->SetStretchBltMode( COLORONCOLOR);
pDC->StretchBlt(0, 0,rc.Width(),rc.Height() , &MemDC, 0, 0, bm.bmWidth, bm.bmHeight, SRCCOPY);
MemDC.SelectObject(gob);
DeleteObject(pBitmap);
DeleteObject(MemDC);
DeleteObject(&bm);
ReleaseDC(wnd->GetSafeHwnd(), MemDC);
}
void initDecoder()
{
m_avCodecContext = avcodec_alloc_context();
if (!m_avCodecContext)
{
//failed to allocate codec context
Cleanup();
return;
}
m_avCodecContext->flags = 0;
uint8_t startCode[] = { 0x00, 0x00, 0x01 };
//////////////////////////////////////////////////////////////////////////
//I thought for play live video you can comment these lines.
if (m_sProps != NULL)
{
// USES_CONVERSION;
// ::MessageBox(NULL, A2T(sprops), TEXT("sprops"), MB_OK);
unsigned spropCount;
SPropRecord* spropRecords = parseSPropParameterSets(m_sProps, spropCount);
try
{
for (unsigned i = 0; i < spropCount; ++i)
{
AddExtraData(startCode, sizeof(startCode));
AddExtraData(spropRecords[i].sPropBytes, spropRecords[i].sPropLength);
}
}
catch (void*)
{
//extradata exceeds size limit
delete[] spropRecords;
Cleanup();
return;
}
delete[] spropRecords;
m_avCodecContext->extradata = extraDataBuffer;
m_avCodecContext->extradata_size = extraDataSize;
}
AddExtraData(startCode, sizeof(startCode));
bInitEx = true;
av_register_all();
avcodec_register_all();
m_codecId = CODEC_ID_H264;
m_avCodec = avcodec_find_decoder(m_codecId);
if (m_avCodec == NULL)
{
return;
}
if (avcodec_open(m_avCodecContext, m_avCodec) < 0)
{
//failed to open codec
Cleanup();
return;
}
if (m_avCodecContext->codec_id == CODEC_ID_H264)
{
m_avCodecContext->flags2 |= CODEC_FLAG2_CHUNKS;
//avCodecContext->flags2 |= CODEC_FLAG2_SHOW_ALL;
}
m_avFrame = avcodec_alloc_frame();
if (!m_avFrame)
{
//failed to allocate frame
Cleanup();
return;
}
}