Saving Byte Array to a RAW file format - c++

I have a simple program that reads data from a PNG into a 2D array. I would like to save that data to a .RAW file so that Raw Studio or Irfanview can view the raw image that my program outputs to my_out.raw. Currently if I just write the raw binary data to the my_out.raw file, neither application can actually read the file, that is view the image. What do I need to do to the program below so that I can see the image?
The code to read the PNG files is:
// MAIN.cpp
#include "pngfilereader.h"
#include <string>
#include <vector>
#include <fstream>
int main (int argc, char *argv[])
{
PNGFileReader pngfr;
if (!pngfr.decompress_png_to_raw(std::string("/home/matt6809/Downloads"
"/City.png"))) {
std::cout << "File decompression error: " << std::endl;
} else {
std::ofstream out;
out.open("./my_out.raw", std::ios_base::out);
std::vector<std::vector<unsigned char> > data;
pngfr.get_image_data(data);
typedef std::vector<std::vector<unsigned char> >::iterator row_it;
typedef std::vector<unsigned char>::iterator col_it;
for(row_it rit= data.begin(); rit != data.end(); ++rit) {
for(col_it cit = rit->begin(); cit != rit->end(); ++cit) {
out << (*cit);
}
}
out << std::endl;
}
return 0;
}
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <png.h>
#include <iostream>
#include <vector>
#include <string>
class PNGFileReader
{
public:
PNGFileReader();
~PNGFileReader();
// Public exposed API:
bool compress_raw_to_png(uint8_t data, int size);
bool decompress_png_to_raw(const std::string &path);
// Getters
long unsigned int get_image_width();
long unsigned int get_image_height();
void get_image_data(std::vector<std::vector<unsigned char> > &data);
private:
// Helper functions:
bool read_png(const std::string &path);
bool create_png_structs(FILE *fp);
bool free_data();
bool alloc_data();
// Member variables:
png_structp m_pPNG;
png_infop m_pPNGInfo;
png_infop m_pPNGEndInfo;
png_bytepp m_Data;
long unsigned int m_ImageWidth;
long unsigned int m_ImageHeight;
// Enums
enum PNGBOOL {NOT_PNG, PNG};
enum PNGERRORS {ERROR, SUCCESS};
};
#include "pngfilereader.h"
#include <stdexcept>
PNGFileReader::PNGFileReader() :
m_pPNG(NULL),
m_pPNGInfo(NULL),
m_pPNGEndInfo(NULL),
m_Data(NULL),
m_ImageWidth(0),
m_ImageHeight(0)
{
}
PNGFileReader::~PNGFileReader()
{
for (unsigned long int i = 0; i < m_ImageHeight; ++i) {
if (m_Data[i]) {
delete m_Data[i];
m_Data[i] = NULL;
}
}
if (m_Data) {
delete m_Data;
m_Data = NULL;
}
}
// Public Exposed API
bool PNGFileReader::compress_raw_to_png(uint8_t m_Data, int size)
{
return PNGFileReader::SUCCESS;
}
bool PNGFileReader::decompress_png_to_raw(const std::string &path)
{
return read_png(path);
}
// Getters
long unsigned int PNGFileReader::get_image_width()
{
return m_ImageWidth;
}
long unsigned int PNGFileReader::get_image_height()
{
return m_ImageHeight;
}
void PNGFileReader::get_image_data(
std::vector<std::vector<unsigned char> > &data)
{
for (unsigned long int i = 0; i < m_ImageHeight; ++i) {
std::vector<unsigned char> v;
data.push_back(v);
for (unsigned long int j = 0; j < m_ImageWidth; ++j) {
std::vector<unsigned char> *vp = &data[i];
vp->push_back(m_Data[i][j]);
}
}
}
// Private Methods
bool PNGFileReader::read_png(const std::string &path)
{
/*
* Open up the file to read (path) in binary mode
* first so that if anything goes wrong with libpng
* we won't have much to undo
*/
const char *c_path = path.c_str();
FILE *fp = fopen(c_path, "rb");
if (!fp)
return PNGFileReader::ERROR;
/*
* Read the first BYTES_TO_READ bytes from file
* then determine if it is a png file or
* not. If png_sig_cmp == 0 all is okay
*/
enum {BYTES_TO_READ = 8};
unsigned char sig[BYTES_TO_READ];
if (!fread(sig, 1, BYTES_TO_READ, fp)) {
fclose(fp);
return PNGFileReader::ERROR;
}
bool is_png = !png_sig_cmp(sig, 0, BYTES_TO_READ);
if (!is_png) {
fclose(fp);
return PNGFileReader::ERROR;
}
if (!this->create_png_structs(fp)) {
fclose(fp);
return PNGFileReader::ERROR;
}
/*
* For error handling purposes. Set a long pointer
* back to this function to handle all error related
* to file IO
*/
if (setjmp(png_jmpbuf(m_pPNG)))
{
png_destroy_read_struct(&m_pPNG, &m_pPNGInfo, &m_pPNGEndInfo);
fclose(fp);
return PNGFileReader::ERROR;
}
/*
* Set up the input code for FILE openend in binary mode,
* and tell libpng we have already read BYTES_TO_READ btyes from
* signature
*/
png_init_io(m_pPNG, fp);
png_set_sig_bytes(m_pPNG, BYTES_TO_READ);
/*
* Using the lowlevel interface to lib png ...
*/
png_read_info(m_pPNG, m_pPNGInfo);
m_ImageHeight = png_get_image_height(m_pPNG, m_pPNGInfo);
m_ImageWidth = png_get_rowbytes(m_pPNG, m_pPNGInfo);
this->alloc_data();
png_read_image(m_pPNG, m_Data);
png_read_end(m_pPNG, NULL);
png_destroy_read_struct(&m_pPNG, &m_pPNGInfo, &m_pPNGEndInfo);
fclose(fp);
return PNGFileReader::SUCCESS;
}
bool PNGFileReader::create_png_structs(FILE *fp)
{
/*
* Create the pointer to main libpng struct, as well as
* two info structs to maintain information after, and
* prior to all operations on png m_Data. Only necessary
* to release resource after function succeeds.
*/
m_pPNG = png_create_read_struct(PNG_LIBPNG_VER_STRING, (png_voidp)NULL,
NULL, NULL);
if (!m_pPNG)
{
fclose(fp);
return PNGFileReader::ERROR;
}
m_pPNGInfo = png_create_info_struct(m_pPNG);
if (!m_pPNGInfo)
{
png_destroy_read_struct(&m_pPNG, (png_infopp)NULL,(png_infopp)NULL);
fclose(fp);
return PNGFileReader::ERROR;
}
m_pPNGEndInfo = png_create_info_struct(m_pPNG);
if (!m_pPNGEndInfo)
{
png_destroy_read_struct(&m_pPNG, &m_pPNGInfo, (png_infopp)NULL);
fclose(fp);
return PNGFileReader::ERROR;
}
return PNGFileReader::SUCCESS;
}
bool PNGFileReader::free_data()
{
if (m_ImageHeight == 0 || m_ImageWidth == 0)
return PNGFileReader::ERROR;
for (unsigned long int i = 0; i < m_ImageHeight; ++i) {
if (m_Data[i]) {
delete m_Data[i];
m_Data[i] = NULL;
}
}
if (m_Data) {
delete m_Data;
m_Data = NULL;
}
return PNGFileReader::SUCCESS;
}
bool PNGFileReader::alloc_data()
{
if (m_ImageHeight == 0 || m_ImageWidth == 0)
return PNGFileReader::ERROR;
if (m_Data != NULL)
this->free_data();
m_Data = new png_bytep[m_ImageHeight]();
for (unsigned long int i = 0; i < m_ImageHeight; ++i) {
m_Data[i] = NULL;
}
try {
for (unsigned long int i = 0; i < m_ImageHeight; ++i) {
m_Data[i] = new png_byte[m_ImageWidth];
}
}
catch (std::bad_alloc e) {
for (unsigned long int i = 0; i < m_ImageHeight; ++i) {
if (m_Data[i]) {
delete m_Data[i];
m_Data[i] = NULL;
}
}
if (m_Data) {
delete m_Data;
m_Data = NULL;
}
throw e;
}
return PNGFileReader::SUCCESS;
}

A "raw" file that is intended to be used with a camera-image processing program like Raw Studio and Irfraview is not a raw-binary dump of the image-data with no header. Instead the "raw" moniker refers to the fact that the image has a minimal amount of image-processing applied in-camera. For instance, the image-data may still be a single-channel monochrome image from the camera's bayer-pattern CFA, or no white-balance, color-matrix, etc. has been applied, etc. Either way, the image-data is still formatted in a standard binary image file format complete with a header, data-packing method, etc. Examples include formats such as Adobe's DNG file format (which is based on TIFF), or proprietary formats from camera manufacturer's themselves such as Canon's CR2, Nikon's NEF, etc.
So if you want these raw-file processing programs to read your "raw" file image data, you'll have to read-up on the binary data specifications the raw-file formats they support, and then re-format the original PNG image-data correctly.

Related

bsoncxx - is there a way to iterate through documents in .bson dump file?

Similarly to how we construct bsoncxx::document::view objects from a buffer with a single binary document, is there a way to extract single documents from a collection in a .bson dump in this framework without having to load them into a DB?
i.e. what works for single document objects
uint8 *buffer; // single bson document
size_t length; // size of buffer
bsoncxx::document::view view(buffer, length);
for (auto elem : view) {
doSomethingWithElem()
}
I'd like to be able to construct a cursor for the whole dump, but without loading it into a collection. Is something like this possible?
Found the solution and it was pretty simple in the end - I utilized the libbson library.
An example of what I used below:
#include <bson.h>
// and other includes
void read_bson() {
bson_reader_t *reader;
const bson_t *doc;
bson_error_t error;
bool eof;
char *path;
reader = bson_reader_new_from_file(path, &error);
if (reader)
{
while ((doc = bson_reader_read(reader, &eof)))
{
const uint8_t *buffer = bson_get_data(doc);
auto view = bsoncxx::document::view(buffer, doc->len);
}
}
}
If you are using mongo-cxx-driver, here is a example for reading bson file dumped by mongodump tool.
#include <iostream>
#include <vector>
#include <fstream>
#include <mongocxx/client.hpp>
#include <mongocxx/uri.hpp>
#include <bsoncxx/json.hpp>
#include <bsoncxx/builder/stream/document.hpp>
#include <sstream>
void parse_one_doc(const char *data, const size_t &length) {
// TODO fill your code
bsoncxx::document::view view((std::uint8_t *) data, length);
std::cout << bsoncxx::to_json(view) << std::endl;
}
int unpack_size(const char *data, size_t position) {
return *(int *) (data + position);
}
bool parse_mongo_dumper(const std::string &data) {
size_t position = 0u, end = data.length() - 1u, data_len = data.size();
size_t obj_size, obj_end;
const char *dc = data.c_str();
while (position < end) {
obj_size = unpack_size(dc, position);
if (position + obj_size > data_len) {
return false;
}
obj_end = position + obj_size;
if (*(dc + obj_end - 1) != '\0') {
return false;
}
parse_one_doc(dc + position, obj_size);
position = obj_end;
}
return true;
}
int main() {
std::string f = "/path/to/data.bson";
// read all data into string
std::ifstream t(f);
std::stringstream buffer;
buffer << t.rdbuf();
std::string s = buffer.str();
// parse bson
parse_mongo_dumper(s);
return 0;
}

How to asynchronously read/write in C++?

How do you copy one stream to another using dedicated read/write threads in C++?
Let's say I have these methods (not real, but to illustrate the point) to read/write data from. These read/write functions could represent anything (network/file/USB/serial/etc).
// returns the number of bytes read
void read(char* buffer, int bufferSize, int* bytesRead);
// returns the number of bytes written
void write(char* buffer, int bufferSize, int* bytesWritten);
The solution should also be portable.
NOTE: I am aware that Windows has a FILE_FLAG_OVERLAPPED feature, but this assumes that the read/write is file IO. Remember, these read/write methods could represent anything.
Here is the solution I came up with.
Header
#pragma once
#include <stdlib.h>
#include <queue>
#include <mutex>
#include <thread>
#include <chrono>
#include <list>
#include <thread>
#define ASYNC_COPY_READ_WRITE_SUCCESS 0
struct BufferBlock;
struct ReadStream
{
// read a stream to a buffer.
// return non-zero if error occured
virtual int read(char* buffer, int bufferSize, int* bytesRead) = 0;
};
struct WriteStream
{
// write a buffer to a stream.
// return non-zero if error occured
virtual int write(char* buffer, int bufferSize, int* bytesWritten) = 0;
};
class BufferBlockManager
{
public:
BufferBlockManager(int numberOfBlocks, int bufferSize);
~BufferBlockManager();
void enqueueBlockForRead(BufferBlock* block);
void dequeueBlockForRead(BufferBlock** block);
void enqueueBlockForWrite(BufferBlock* block);
void dequeueBlockForWrite(BufferBlock** block);
void resetState();
private:
std::list<BufferBlock*> blocks;
std::queue<BufferBlock*> blocksPendingRead;
std::queue<BufferBlock*> blocksPendingWrite;
std::mutex queueLock;
std::chrono::milliseconds dequeueSleepTime;
};
void AsyncCopyStream(BufferBlockManager* bufferBlockManager, ReadStream* readStream, WriteStream* writeStream, int* readResult, int* writeResult);
CPP
#include "AsyncReadWrite.h"
struct BufferBlock
{
BufferBlock(int bufferSize) : buffer(NULL)
{
this->bufferSize = bufferSize;
this->buffer = new char[bufferSize];
this->actualSize = 0;
this->isLastBlock = false;
}
~BufferBlock()
{
this->bufferSize = 0;
free(this->buffer);
this->buffer = NULL;
this->actualSize = 0;
}
char* buffer;
int bufferSize;
int actualSize;
bool isLastBlock;
};
BufferBlockManager::BufferBlockManager(int numberOfBlocks, int bufferSize)
{
dequeueSleepTime = std::chrono::milliseconds(100);
for (int x = 0; x < numberOfBlocks; x++)
{
BufferBlock* block = new BufferBlock(bufferSize);
blocks.push_front(block);
blocksPendingRead.push(block);
}
}
BufferBlockManager::~BufferBlockManager()
{
for (std::list<BufferBlock*>::const_iterator iterator = blocks.begin(), end = blocks.end(); iterator != end; ++iterator) {
delete (*iterator);
}
}
void BufferBlockManager::enqueueBlockForRead(BufferBlock* block)
{
queueLock.lock();
block->actualSize = 0;
block->isLastBlock = false;
blocksPendingRead.push(block);
queueLock.unlock();
}
void BufferBlockManager::dequeueBlockForRead(BufferBlock** block)
{
WAITFOR:
while (blocksPendingRead.size() == 0)
std::this_thread::sleep_for(dequeueSleepTime);
queueLock.lock();
if (blocksPendingRead.size() == 0)
{
queueLock.unlock();
goto WAITFOR;
}
*block = blocksPendingRead.front();
blocksPendingRead.pop();
queueLock.unlock();
}
void BufferBlockManager::enqueueBlockForWrite(BufferBlock* block)
{
queueLock.lock();
blocksPendingWrite.push(block);
queueLock.unlock();
}
void BufferBlockManager::dequeueBlockForWrite(BufferBlock** block)
{
WAITFOR:
while (blocksPendingWrite.size() == 0)
std::this_thread::sleep_for(dequeueSleepTime);
queueLock.lock();
if (blocksPendingWrite.size() == 0)
{
queueLock.unlock();
goto WAITFOR;
}
*block = blocksPendingWrite.front();
blocksPendingWrite.pop();
queueLock.unlock();
}
void BufferBlockManager::resetState()
{
queueLock.lock();
blocksPendingRead = std::queue<BufferBlock*>();
blocksPendingWrite = std::queue<BufferBlock*>();
for (std::list<BufferBlock*>::const_iterator iterator = blocks.begin(), end = blocks.end(); iterator != end; ++iterator) {
(*iterator)->actualSize = 0;
}
queueLock.unlock();
}
struct AsyncCopyContext
{
AsyncCopyContext(BufferBlockManager* bufferBlockManager, ReadStream* readStream, WriteStream* writeStream)
{
this->bufferBlockManager = bufferBlockManager;
this->readStream = readStream;
this->writeStream = writeStream;
this->readResult = ASYNC_COPY_READ_WRITE_SUCCESS;
this->writeResult = ASYNC_COPY_READ_WRITE_SUCCESS;
}
BufferBlockManager* bufferBlockManager;
ReadStream* readStream;
WriteStream* writeStream;
int readResult;
int writeResult;
};
void ReadStreamThread(AsyncCopyContext* asyncContext)
{
int bytesRead = 0;
BufferBlock* readBuffer = NULL;
int readResult = ASYNC_COPY_READ_WRITE_SUCCESS;
while (
// as long there hasn't been any write errors
asyncContext->writeResult == ASYNC_COPY_READ_WRITE_SUCCESS
// and we haven't had an error reading yet
&& readResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
// let's deque a block to read to!
asyncContext->bufferBlockManager->dequeueBlockForRead(&readBuffer);
readResult = asyncContext->readStream->read(readBuffer->buffer, readBuffer->bufferSize, &bytesRead);
readBuffer->actualSize = bytesRead;
readBuffer->isLastBlock = bytesRead == 0;
if (readResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
// this was a valid read, go ahead and queue it for writing
asyncContext->bufferBlockManager->enqueueBlockForWrite(readBuffer);
}
else
{
// an error occured reading
asyncContext->readResult = readResult;
// since an error occured, lets queue an block to write indicatiting we are done and there are no more bytes to read
readBuffer->isLastBlock = true;
readBuffer->actualSize = 0;
asyncContext->bufferBlockManager->enqueueBlockForWrite(readBuffer);
}
if (readBuffer->isLastBlock) return;
}
}
void WriteStreamThread(AsyncCopyContext* asyncContext)
{
int bytesWritten = 0;
BufferBlock* writeBuffer = NULL;
int writeResult = ASYNC_COPY_READ_WRITE_SUCCESS;
bool isLastWriteBlock = false;
while (
// as long as there are no errors during reading
asyncContext->readResult == ASYNC_COPY_READ_WRITE_SUCCESS
// and we haven't had an error writing yet
&& writeResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
// lets dequeue a block for writing!
asyncContext->bufferBlockManager->dequeueBlockForWrite(&writeBuffer);
isLastWriteBlock = writeBuffer->isLastBlock;
if (writeBuffer->actualSize > 0)
writeResult = asyncContext->writeStream->write(writeBuffer->buffer, writeBuffer->actualSize, &bytesWritten);
if (writeResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
asyncContext->bufferBlockManager->enqueueBlockForRead(writeBuffer);
if (isLastWriteBlock) return;
}
else
{
asyncContext->writeResult = writeResult;
asyncContext->bufferBlockManager->enqueueBlockForRead(writeBuffer);
return;
}
}
}
void AsyncCopyStream(BufferBlockManager* bufferBlockManager, ReadStream* readStream, WriteStream* writeStream, int* readResult, int* writeResult)
{
AsyncCopyContext asyncContext(bufferBlockManager, readStream, writeStream);
std::thread readThread(ReadStreamThread, &asyncContext);
std::thread writeThread(WriteStreamThread, &asyncContext);
readThread.join();
writeThread.join();
*readResult = asyncContext.readResult;
*writeResult = asyncContext.writeResult;
}
Usage
#include <stdio.h>
#include <tchar.h>
#include "AsyncReadWrite.h"
struct ReadTestStream : ReadStream
{
int readCount = 0;
int read(char* buffer, int bufferSize, int* bytesRead)
{
printf("Starting read...\n");
memset(buffer, bufferSize, 0);
if (readCount == 10)
{
*bytesRead = 0;
return 0;
}
// pretend this function takes a while!
std::this_thread::sleep_for(std::chrono::milliseconds(100));
char buff[100];
sprintf_s(buff, "This is read number %d\n", readCount);
strcpy_s(buffer, sizeof(buff), buff);
*bytesRead = strlen(buffer);
readCount++;
printf("Finished read...\n");
return 0;
}
};
struct WriteTestStream : WriteStream
{
int write(char* buffer, int bufferSize, int* bytesWritten)
{
printf("Starting write...\n");
// pretend this function takes a while!
std::this_thread::sleep_for(std::chrono::milliseconds(500));
printf(buffer);
printf("Finished write...\n");
return 0;
}
};
int _tmain(int argc, _TCHAR* argv[])
{
BufferBlockManager bufferBlockManager(5, 4096);
ReadTestStream readStream;
WriteTestStream writeStream;
int readResult = 0;
int writeResult = 0;
printf("Starting copy...\n");
AsyncCopyStream(&bufferBlockManager, &readStream, &writeStream, &readResult, &writeResult);
printf("Finished copy... readResult=%d writeResult=%d \n", readResult, writeResult);
getchar();
return 0;
}
EDIT: I put my solution into a GitHub repository here. If you wish to use this code, refer to the repository since it may be more updated than this answer.
Typically, you would just have one thread for each direction that alternates between reads and writes.

Decoding by libjpeg -> Encoding by x264, strange artefacts on frames

I have a collection of jpeg, which must be decoded by lib jpeg, and after it, encoded by x264 (after it encoded packets are streamed via rtmp).
Code I used for decoding:
struct my_error_mgr
{
struct jpeg_error_mgr pub;
jmp_buf setjmp_buffer;
};
typedef my_error_mgr *my_error_ptr;
METHODDEF(void) my_error_exit (j_common_ptr cinfo)
{
my_error_ptr myerr = (my_error_ptr) cinfo->err;
(*cinfo->err->output_message) (cinfo);
longjmp(myerr->setjmp_buffer, 1);
}
void init_source(j_decompress_ptr ptr)
{
Q_UNUSED(ptr)
}
boolean fill_input_buffer(j_decompress_ptr ptr)
{
Q_UNUSED(ptr)
return TRUE;
}
void term_source(j_decompress_ptr ptr)
{
Q_UNUSED(ptr)
}
void skip_input_data(j_decompress_ptr ptr, long num_bytes)
{
if(num_bytes>0)
{
ptr->src->next_input_byte+=(size_t)num_bytes;
ptr->src->bytes_in_buffer-=(size_t)num_bytes;
}
}
EtherDecoder::EtherDecoder(QObject *parent):
QObject(parent)
{
}
void EtherDecoder::dataBlockReady(QByteArray data)
{
jpeg_decompress_struct decompressInfo;
jpeg_create_decompress(&decompressInfo);
my_error_mgr err;
decompressInfo.do_fancy_upsampling = FALSE;
decompressInfo.src = (jpeg_source_mgr *) (*decompressInfo.mem->alloc_small) ((j_common_ptr) &decompressInfo, JPOOL_PERMANENT, sizeof(jpeg_source_mgr));
decompressInfo.err = jpeg_std_error(&err.pub);
err.pub.error_exit = my_error_exit;
if (setjmp(err.setjmp_buffer))
{
jpeg_destroy_decompress(&decompressInfo);
return;
}
decompressInfo.src->init_source = init_source;
decompressInfo.src->resync_to_restart = jpeg_resync_to_restart;
decompressInfo.src->fill_input_buffer = fill_input_buffer;
decompressInfo.src->skip_input_data = skip_input_data;
decompressInfo.src->term_source = term_source;
decompressInfo.src->next_input_byte = reinterpret_cast<const JOCTET*>(data.data());
decompressInfo.src->bytes_in_buffer = data.size();
jpeg_read_header(&decompressInfo, TRUE);
jpeg_start_decompress(&decompressInfo);
int size = 0;
int n_samples = 0;
char *samples = new char[5242880];
char *reserv = samples;
while (decompressInfo.output_scanline < decompressInfo.output_height)
{
n_samples = jpeg_read_scanlines(&decompressInfo, (JSAMPARRAY) &samples, 1);
samples += n_samples * decompressInfo.image_width * decompressInfo.num_components;
size += n_samples * decompressInfo.image_width * decompressInfo.num_components;
}
jpeg_finish_decompress(&decompressInfo);
QByteArray output(reserv, size);
emit frameReady(output, decompressInfo.output_width, decompressInfo.output_height);
jpeg_destroy_decompress(&decompressInfo);
delete[] reserv;
}
When I emit frameReady signal, I send data to Encoder, method, where I init Encedor looks like:
bool EtherEncoder::initEncoder(unsigned int width, unsigned int height)
{
x264_param_t param;
x264_param_default_preset(&param, "veryfast", "zerolatency");
param.i_width=width;
param.i_height=height;
param.i_frame_total=0;
param.i_csp=X264_CSP_I420;
param.i_timebase_num=1;
param.i_timebase_den=96000;
param.b_annexb=true;
param.b_repeat_headers=false;
x264_param_apply_fastfirstpass(&param);
x264_param_apply_profile(&param, "baseline");
_context=x264_encoder_open(&param);
if(!_context)
return false;
int nal_count;
x264_nal_t *nals;
if(x264_encoder_headers(_context, &nals, &nal_count)<0)
{
x264_encoder_close(_context);
_context=0;
return false;
}
_extradata=QByteArray();
_width=width;
_height=height;
if(nal_count>0)
{
_extradata=QByteArray(
(const char *)nals[0].p_payload,
nals[nal_count-1].p_payload+nals[nal_count-1].i_payload-nals[0].p_payload);
}
return true;
}
And encoding method:
void EtherEncoder::onFrameReady(QByteArray data, int width, int height)
{
while(data.size()>0)
{
if(!_context && initEncoder(width, height))
{
_timestampDelta=realTimestamp();
}
if(_context)
{
x264_picture_t pic;
x264_picture_init(&pic);
pic.i_type=X264_TYPE_AUTO;
pic.i_pts=_timestampDelta*96000;
pic.img.i_csp=X264_CSP_I420;
pic.img.i_plane=3;
int planeSize = width*height;
uint8_t *p = (uint8_t*)data.data();
pic.img.plane[0]=p;
p+=planeSize;
pic.img.plane[1]=p;
p+=planeSize/4;
pic.img.plane[2]=p;
pic.img.i_stride[0]=width;
pic.img.i_stride[1]=width/2;
pic.img.i_stride[2]=width/2;
if(_forceKeyFrame)
{
pic.i_type=X264_TYPE_I;
_forceKeyFrame=false;
}
int nal_count;
x264_nal_t *nals;
int rc=x264_encoder_encode(_context, &nals, &nal_count, &pic, &pic);
if(rc>0)
{
_mutex.lock();
_packets.push_back(
Packet(
QByteArray(
(const char *)nals[0].p_payload, nals[nal_count- 1].p_payload+nals[nal_count-1].i_payload-nals[0].p_payload),
_timestampDelta/96.0,
_timestampDelta/96.0,
pic.b_keyframe));
_timestampDelta+=40;
data.clear();
_mutex.unlock();
emit onPacketReady();
}
}
}
}
Decoding and encoding proceeds without errors, at the end I get valid video stream, but, it seems that in one of this steps I set Invalid data for decoder/encoder. I get only 1/4 part of image (top-left, as I understood) and it has invalid color and come color stripes. Maybe I set invalid strides and planes when encode frame, or maybe my setting data for libjpeg decoder is incorrect.. Please ask questions about my code, I'll try to make some explanations for you. I explodes my brain.. Thank you.

Random Bytes added to an end of a buffer

I was making a re-creation of some System.IO functions from that class.
When I setup a buffer and allocate n number of bytes it'll read bytes to that and then add random bytes to the end of that buffer.
For example:
My Main:
int main(int argc, char *args[])
{
SetConsoleTitle(TEXT("Stream Test."));
cout<<"Press any Key to begin reading.";
cin.get();
const char* data = File::ReadAllBytes(args[1]);
Stream* stream = new Stream(data);
char* magic = new char[8];
stream->Read(magic, 0, 8);
magic[8] = '\0';
cout<<magic<<endl<<endl;
delete[]data;
cout<<"Press any key to quit.";
cin.get();
return 0;
}
and here is my System::IO namespace + stream class:
namespace System
{
namespace IO
{
class File
{
public:
static char* ReadAllBytes(const char *name)
{
ifstream fl(name, ifstream::in|ifstream::binary);
fl.seekg( 0, ifstream::end );
size_t len = fl.tellg();
char* ret = new char[len+1];
ret[len] = '\0';
fl.seekg(0);
fl.read(ret, len);
fl.close();
return ret;
}
//not sure of this use yet.
static size_t fileSize(const char* filename)
{
ifstream in(filename, ifstream::in | ifstream::binary);
in.seekg(0, ifstream::end);
return in.tellg();
}
};
class Stream
{
public:
const char *_buffer;
__int64 _origin;
__int64 _position;
__int64 _length;
__int64 _capacity;
bool _expandable;
bool _writable;
bool _exposable;
bool _isOpen;
static const int MemStreamMaxLength = 2147483647;
Stream()
{
InitializeInstanceFields();
}
Stream(const char *buffer)
{
_buffer = buffer;
_length = strlen(_buffer);
_capacity = _length;
_position = 0;
_origin = 0;
_expandable = false;
_writable = true;
_exposable = true;
_isOpen = true;
}
int ReadByte()
{
if (_position >= _length)
return -1;
return _buffer[_position++];
}
void Read(char* &buffer, int offset, int length)
{
if((_position + offset + length) <= _length)
{
memcpy( buffer, _buffer + (_position + offset), length );
_position += length;
}
}
private:
void InitializeInstanceFields()
{
_origin = 0;
_position = 0;
_length = 0;
_capacity = 0;
_expandable = false;
_writable = false;
_exposable = false;
_isOpen = false;
}
};
}
}
This is what ends up happening:
Can anyone explain why this happens, how I can fix, or anything else? I'm new to C++ so any explanations would help. Also please don't criticize my scripting, I know it may be bad, outdated, deprecated, etc. but I'm open to learning and any helping advice goes for the better. :)
You can only use operator << (char *) on C-style strings, not arbitrary arrays of characters. How would you expect it to know how many characters to output?
I would guess the file was not opened correctly and thus the magic buffer is not set at all which leaves it with initialized junk data:
If the constructor is not successful in opening the file, the object
is still created although no file is associated to the stream buffer
and the stream's failbit is set (which can be checked with inherited
member fail).
http://www.cplusplus.com/reference/fstream/ifstream/ifstream/
Try adding more error checking along the way (using cout), especially when opening and reading the buffer. Perhaps set the magic buffer to zero or something recognizable that is overwritten when successful.

Generate SHA hash in C++ using OpenSSL library

How can I generate SHA1 or SHA2 hashes using the OpenSSL libarary?
I searched google and could not find any function or example code.
From the command line, it's simply:
printf "compute sha1" | openssl sha1
You can invoke the library like this:
#include <stdio.h>
#include <string.h>
#include <openssl/sha.h>
int main()
{
unsigned char ibuf[] = "compute sha1";
unsigned char obuf[20];
SHA1(ibuf, strlen(ibuf), obuf);
int i;
for (i = 0; i < 20; i++) {
printf("%02x ", obuf[i]);
}
printf("\n");
return 0;
}
OpenSSL has a horrible documentation with no code examples, but here you are:
#include <openssl/sha.h>
bool simpleSHA256(void* input, unsigned long length, unsigned char* md)
{
SHA256_CTX context;
if(!SHA256_Init(&context))
return false;
if(!SHA256_Update(&context, (unsigned char*)input, length))
return false;
if(!SHA256_Final(md, &context))
return false;
return true;
}
Usage:
unsigned char md[SHA256_DIGEST_LENGTH]; // 32 bytes
if(!simpleSHA256(<data buffer>, <data length>, md))
{
// handle error
}
Afterwards, md will contain the binary SHA-256 message digest. Similar code can be used for the other SHA family members, just replace "256" in the code.
If you have larger data, you of course should feed data chunks as they arrive (multiple SHA256_Update calls).
Adaptation of #AndiDog version for big file:
static const int K_READ_BUF_SIZE{ 1024 * 16 };
std::optional<std::string> CalcSha256(std::string filename)
{
// Initialize openssl
SHA256_CTX context;
if(!SHA256_Init(&context))
{
return std::nullopt;
}
// Read file and update calculated SHA
char buf[K_READ_BUF_SIZE];
std::ifstream file(filename, std::ifstream::binary);
while (file.good())
{
file.read(buf, sizeof(buf));
if(!SHA256_Update(&context, buf, file.gcount()))
{
return std::nullopt;
}
}
// Get Final SHA
unsigned char result[SHA256_DIGEST_LENGTH];
if(!SHA256_Final(result, &context))
{
return std::nullopt;
}
// Transform byte-array to string
std::stringstream shastr;
shastr << std::hex << std::setfill('0');
for (const auto &byte: result)
{
shastr << std::setw(2) << (int)byte;
}
return shastr.str();
}
correct syntax at command line should be
echo -n "compute sha1" | openssl sha1
otherwise you'll hash the trailing newline character as well.
Here is OpenSSL example of calculating sha-1 digest using BIO:
#include <openssl/bio.h>
#include <openssl/evp.h>
std::string sha1(const std::string &input)
{
BIO * p_bio_md = nullptr;
BIO * p_bio_mem = nullptr;
try
{
// make chain: p_bio_md <-> p_bio_mem
p_bio_md = BIO_new(BIO_f_md());
if (!p_bio_md) throw std::bad_alloc();
BIO_set_md(p_bio_md, EVP_sha1());
p_bio_mem = BIO_new_mem_buf((void*)input.c_str(), input.length());
if (!p_bio_mem) throw std::bad_alloc();
BIO_push(p_bio_md, p_bio_mem);
// read through p_bio_md
// read sequence: buf <<-- p_bio_md <<-- p_bio_mem
std::vector<char> buf(input.size());
for (;;)
{
auto nread = BIO_read(p_bio_md, buf.data(), buf.size());
if (nread < 0) { throw std::runtime_error("BIO_read failed"); }
if (nread == 0) { break; } // eof
}
// get result
char md_buf[EVP_MAX_MD_SIZE];
auto md_len = BIO_gets(p_bio_md, md_buf, sizeof(md_buf));
if (md_len <= 0) { throw std::runtime_error("BIO_gets failed"); }
std::string result(md_buf, md_len);
// clean
BIO_free_all(p_bio_md);
return result;
}
catch (...)
{
if (p_bio_md) { BIO_free_all(p_bio_md); }
throw;
}
}
Though it's longer than just calling SHA1 function from OpenSSL, but it's more universal and can be reworked for using with file streams (thus processing data of any length).
C version of #Nayfe code, generating SHA1 hash from file:
#include <stdio.h>
#include <openssl/sha.h>
static const int K_READ_BUF_SIZE = { 1024 * 16 };
unsigned char* calculateSHA1(char *filename)
{
if (!filename) {
return NULL;
}
FILE *fp = fopen(filename, "rb");
if (fp == NULL) {
return NULL;
}
unsigned char* sha1_digest = malloc(sizeof(char)*SHA_DIGEST_LENGTH);
SHA_CTX context;
if(!SHA1_Init(&context))
return NULL;
unsigned char buf[K_READ_BUF_SIZE];
while (!feof(fp))
{
size_t total_read = fread(buf, 1, sizeof(buf), fp);
if(!SHA1_Update(&context, buf, total_read))
{
return NULL;
}
}
fclose(fp);
if(!SHA1_Final(sha1_digest, &context))
return NULL;
return sha1_digest;
}
It can be used as follows:
unsigned char *sha1digest = calculateSHA1("/tmp/file1");
The res variable contains the sha1 hash.
You can print it on the screen using the following for-loop:
char *sha1hash = (char *)malloc(sizeof(char) * 41);
sha1hash[40] = '\0';
int i;
for (i = 0; i < SHA_DIGEST_LENGTH; i++)
{
sprintf(&sha1hash[i*2], "%02x", sha1digest[i]);
}
printf("SHA1 HASH: %s\n", sha1hash);