Shared memory ring buffer crashes - c++

I wrote a shared memory ringbuffer with QT and I turns out it works great in a single process with one consumer and one writer. If I try to read data from a second process I the first one which connected crashes with a seg fault. Maybe I'm overlooking something?
The Header File
#ifndef SHAREDMEMORYRINGBUFFER_H
#define SHAREDMEMORYRINGBUFFER_H
#include <QObject>
#include <QSharedMemory>
class SharedMemoryRingBuffer : public QObject
{
Q_OBJECT
public:
struct SharedMemoryAttributes {
int32_t readPosition;
int32_t writePosition;
int32_t size;
int32_t* data;
};
explicit SharedMemoryRingBuffer(QString sharedMemoryName, int32_t size, QObject *parent = 0);
~SharedMemoryRingBuffer();
int write(int32_t frame);
int32_t read();
int32_t length();
int32_t readPosition();
int32_t writePosition();
int avaibleSize();
int isEmpty();
void empty();
int isFull();
private:
SharedMemoryAttributes *_attributes;
int32_t _writePosition;
int32_t _readPosition;
int32_t _length;
QSharedMemory *_sharedMemory;
int _headSize;
signals:
void readFrame(QString name);
public slots:
};
#endif // SHAREDMEMORYRINGBUFFER_H
And the CPP File:
#include "sharedmemoryringbuffer.h"
#include <QDebug>
#define DEBUGGINGNAME "[SharedMemoryRingbuffer]"
SharedMemoryRingBuffer::SharedMemoryRingBuffer(QString sharedMemoryName, int32_t size, QObject *parent) : QObject(parent)
{
int sizeOfData = sizeof(int32_t) * (size);
int sizeOfHeader = sizeof(SharedMemoryAttributes);
_sharedMemory = new QSharedMemory(sharedMemoryName);
if (!_sharedMemory->attach()) {
if (!_sharedMemory->create(sizeOfData+sizeOfHeader, QSharedMemory::ReadWrite )) {
qDebug() << DEBUGGINGNAME << "Could not create shared memory object, aborting ...";
qDebug() << DEBUGGINGNAME << _sharedMemory->errorString();
//TODO: exit call
return;
}
}
_sharedMemory->lock();
_attributes = reinterpret_cast<SharedMemoryAttributes*>(_sharedMemory->data());
_attributes->readPosition = 0;
_attributes->writePosition = 0;
_attributes->size = size + 1;
_attributes->data = (int32_t*) _sharedMemory->data() + sizeOfHeader;
_sharedMemory->unlock();
}
SharedMemoryRingBuffer::~SharedMemoryRingBuffer()
{
_sharedMemory->detach();
delete _sharedMemory;
}
int SharedMemoryRingBuffer::write(int32_t frame)
{
_sharedMemory->lock();
if ( !isFull() ) {
_attributes->data[writePosition()] = frame;
if (writePosition() + 1 >= _attributes->size) _attributes->writePosition = 0;
else _attributes->writePosition += 1;
_sharedMemory->unlock();
return 1;
}
_sharedMemory->unlock();
return 0;
}
/**
* #brief SharedMemoryRingBuffer::read
* #return
* if the buffer is empty this functions returns the last readable value
*
*/
int32_t SharedMemoryRingBuffer::read()
{
_sharedMemory->lock();
int32_t frame = _attributes->data[readPosition()];
if ( readPosition() != writePosition() ) {
if (readPosition() + 1 >= _attributes->size ) _attributes->readPosition = 0;
else _attributes->readPosition += 1;
emit readFrame(_sharedMemory->key());
}
_sharedMemory->unlock();
return frame;
}
int32_t SharedMemoryRingBuffer::length()
{
if(readPosition() <= writePosition()) {
return writePosition() - readPosition();
} else {
return readPosition() - writePosition();
}
}
int32_t SharedMemoryRingBuffer::readPosition()
{
return _attributes->readPosition;
}
int32_t SharedMemoryRingBuffer::writePosition()
{
return _attributes->writePosition;
}
int SharedMemoryRingBuffer::avaibleSize()
{
return -1;
}
int SharedMemoryRingBuffer::isEmpty()
{
if ( readPosition() == writePosition() ) return 1;
else return 0;
}
void SharedMemoryRingBuffer::empty()
{
}
int SharedMemoryRingBuffer::isFull()
{
if ((writePosition() + 1) % _attributes->size == readPosition()) return 1;
else return 0;
}

This initialization:
_attributes->size = size + 1;
looks incorrect since the size you pass to _sharedMemory->create() is only allocating enough "data" space for size elements, not size + 1. Why is the +1 there?

So I found a solution which was pretty obvious. If a second process connects to to the shared memory it should NOT set the head and tail position to 0.
This is the fixed code, which works well :)
#include "sharedmemoryringbuffer.h"
#include <QDebug>
#define DEBUGGINGNAME "[SharedMemoryRingbuffer]"
SharedMemoryRingBuffer::SharedMemoryRingBuffer(QString sharedMemoryName, int32_t size, QObject *parent) : QObject(parent)
{
int sizeOfData = sizeof(int32_t) * (size);
int sizeOfHeader = sizeof(SharedMemoryAttributes);
_sharedMemory = new QSharedMemory(sharedMemoryName);
if (_sharedMemory->isAttached()) _sharedMemory->detach();
if (!_sharedMemory->attach()) {
_sharedMemory->lock();
_attributes = reinterpret_cast<SharedMemoryAttributes*>(_sharedMemory->data());
_attributes->readPosition = 0;
_attributes->writePosition = 0;
_attributes->size = size;
_sharedMemory->unlock();
if (!_sharedMemory->create(sizeOfData+sizeOfHeader, QSharedMemory::ReadWrite )) {
qDebug() << DEBUGGINGNAME << "Could not create shared memory object, aborting ...";
qDebug() << DEBUGGINGNAME << _sharedMemory->errorString();
//TODO: exit call
return;
}
}
_attributes->data = (int32_t*) _sharedMemory->data() + sizeOfHeader;
}
SharedMemoryRingBuffer::~SharedMemoryRingBuffer()
{
_sharedMemory->detach();
delete _sharedMemory;
}
int SharedMemoryRingBuffer::write(int32_t frame)
{
_sharedMemory->lock();
if ( !isFull() ) {
_attributes->data[writePosition()] = frame;
if (writePosition() + 1 >= _attributes->size) _attributes->writePosition = 0;
else _attributes->writePosition += 1;
_sharedMemory->unlock();
return 1;
}
_sharedMemory->unlock();
return 0;
}
/**
* #brief SharedMemoryRingBuffer::read
* #return
* if the buffer is empty this functions return the last readable value
*
*/
int32_t SharedMemoryRingBuffer::read()
{
_sharedMemory->lock();
int32_t frame = _attributes->data[readPosition()];
if ( readPosition() != writePosition() ) {
if (readPosition() + 1 >= _attributes->size ) _attributes->readPosition = 0;
else _attributes->readPosition += 1;
emit readFrame(_sharedMemory->key());
}
_sharedMemory->unlock();
return frame;
}
int32_t SharedMemoryRingBuffer::length()
{
if(readPosition() <= writePosition()) {
return writePosition() - readPosition();
} else {
return readPosition() - writePosition();
}
}
int32_t SharedMemoryRingBuffer::readPosition()
{
return _attributes->readPosition;
}
int32_t SharedMemoryRingBuffer::writePosition()
{
return _attributes->writePosition;
}
int SharedMemoryRingBuffer::avaibleSize()
{
return -1;
}
int SharedMemoryRingBuffer::isEmpty()
{
if ( readPosition() == writePosition() ) return 1;
else return 0;
}
void SharedMemoryRingBuffer::empty()
{
}
int SharedMemoryRingBuffer::isFull()
{
if ((writePosition() + 1) % _attributes->size == readPosition()) return 1;
else return 0;
}

Related

Invalid output for a custom print function when incorporating newline characters within a string for a kernel project

I'm in the process of working on a kernel program to handle printing capabilities of input for a custom OS. I'm following Poncho's 2nd YouTube Video series found here, I'm currently on Video 4 in the series where he starts to add numerical types as inputs to the renderer's print function. Now, my code isn't exactly like his as I made some modifications.
-Note- This won't compile directly as there is no main function. _start is being called or invoked by a bootloader that isn't shown here, I will however, add it to the bottom of this question.
When I use the class's print function like this within my kernel:
#include "BasicRenderer.h"
extern "C" void _start(Framebuffer* framebuffer, PSF1_FONT** fonts) {
BasicRenderer = renderer(framebuffer, fonts);
renderer.Print("This is some text");
renderer.Print('\n');
renderer.Print(uint64_t(123456789));
renderer.Print('\n');
renderer.Print(int64_t(-123456789));
return;
}
And I run the kernel in emu. I'm getting the following output displayed:
This is some text
123456789
-123456789
The above is correct, however, when I try to incorporate the ability to parse a newline set of characters being either \n or \0 within of a const char* that acts as a string as seen in the following example:
#include "BasicRenderer.h"
extern "C" void _start(Framebuffer* framebuffer, PSF1_FONT** fonts) {
BasicRenderer = renderer(framebuffer, fonts);
renderer.Print("This is some text\n");
renderer.Print(uint64_t(123456789));
renderer.Print('\n');
renderer.Print(int64_t(-123456789));
return;
}
And now the displayed output is:
This is some text
123456789
-123456789
Here, the output in the second line has a space preceding the numerical value to be displayed after the call to Print() that has a \n within its string. I'm not sure what is causing this in my code. Does it have to do with the while condition or how I'm incrementing and indexing into the character string within BasicRenderer::Print(const char* str)? Or is it coming from BasicRender::PutChar(char c)? Or is it within one of the to_string() functions?
Here is the relevant implementation code...
BasicRenderer.cpp
#include "BasicRenderer.h"
void BasicRenderer::Print(const char* str) {
char* chr = (char*)str;
while(*chr != 0) {
if ( (*chr == '\\') && ((*chr+1 == 'n') || (*chr+1 == '0')) ) {
PutChar('\n');
chr++;
chr++;
} else {
PutChar(*chr);
cursor_position_.x += 8;
if (cursor_position_.x + 8 > framebuffer_->Width) {
cursor_position_.x = 0;
cursor_position_.y += 16;
}
chr++;
}
}
}
void BasicRenderer::Print(uint64_t val) {
const char* str = to_string(val);
Print(str);
}
void BasicRenderer::Print(int64_t val) {
const char* str = to_string(val);
Print(str);
}
void BasicRenderer::PutChar(char c) {
if (c == '\n' || c == '\0') {
cursor_position_.x = 0;
cursor_position_.y += 16;
} else {
unsigned int* pixPtr = (unsigned int*)framebuffer_->BaseAddress;
char* fontPtr = (char*)selected_font_->glyphBuffer + (c * selected_font_->psf1_Header->charsize);
for (unsigned long y = cursor_position_.y; y < cursor_position_.y + 16; y++) {
for (unsigned long x = cursor_position_.x; x < cursor_position_.x + 8; x++) {
if ((*fontPtr & (0b10000000 >> (x - cursor_position_.x))) > 0) {
*(unsigned int*)(pixPtr + x + (y * framebuffer_->PixelsPerScanLine)) = font_color_;
}
}
fontPtr++;
}
}
}
cstr.cpp
#include "cstr.h"
const char* to_string(uint64_t value) {
static char output_uint_buffer[128];
uint8_t size = 0;
uint64_t sizeTest = value;
while (sizeTest / 10 > 0) {
sizeTest /= 10;
size++;
}
uint8_t idx = 0;
while (value / 10 > 0) {
uint8_t remainder = value % 10;
value /= 10;
output_uint_buffer[size - idx] = remainder + '0';
idx++;
}
uint8_t remainder = value % 10;
output_uint_buffer[size-idx] = remainder + '0';
output_uint_buffer[size + 1] = 0;
return output_uint_buffer;
}
const char* to_string(int64_t value) {
static char output_int_buffer[128];
uint8_t isNegative = 0;
if (value < 0) {
isNegative = 1;
value *= -1;
output_int_buffer[0] = '-';
}
uint8_t size = 0;
uint64_t sizeTest = value;
while (sizeTest / 10 > 0) {
sizeTest /= 10;
size++;
}
uint8_t idx = 0;
while (value / 10 > 0) {
uint8_t remainder = value % 10;
value /= 10;
output_int_buffer[isNegative + size - idx] = remainder + '0';
idx++;
}
uint8_t remainder = value % 10;
output_int_buffer[isNegative + size - idx] = remainder + '0';
output_int_buffer[isNegative + size + 1] = 0;
return output_int_buffer;
}
And here is the rest of the declarations...
BasicRender.h
#pragma once
#include "cstr.h"
#include "math.h"
#include "framebuffer.h"
#include "SimpleFonts.h"
class BasicRenderer {
public:
BasicRenderer(Framebuffer* framebuffer, PSF1_FONT** fonts) :
framebuffer_{framebuffer},
fonts_{fonts},
cursor_position_({0,0}),
selected_font_{fonts_[0]},
font_color_{0xFFFFFFFF}
{}
void Print(const char* str);
void Print(char c) { PutChar(c); }
void Print(uint64_t val);
void Print(int64_t val);
private:
void PutChar(char c);
Framebuffer* framebuffer_;
Point cursor_position_;
PSF1_FONT** fonts_;
PSF1_FONT* selected_font_;
unsigned int font_color_;
};
cstr.h
#pragma once
#include <stdint.h>
const char* to_string(uint64_t value);
const char* to_string(int64_t value);
math.h
#pragma once
struct Point {
unsigned int x;
unsigned int y;
};
Framebuffer.h
#pragma once
#include <stddef.h>
struct Framebuffer {
void* BaseAddress;
size_t BufferSize;
unsigned int Width;
unsigned int Height;
unsigned int PixelsPerScanLine;
};
SimpleFonts.h
#pragma once
struct PSF1_HEADER {
unsigned char magic[2];
unsigned char mode;
unsigned char charsize;
};
struct PSF1_FONT {
PSF1_HEADER* psf1_Header;
void* glyphBuffer;
};
Here is the bootloader application that invokes the above kernel.
main.c
#include <efi.h>
#include <efilib.h>
#include <elf.h>
#define PSF1_MAGIC0 0x36
#define PSF1_MAGIC1 0x04
typedef unsigned long long size_t;
typedef struct {
unsigned char magic[2];
unsigned char mode;
unsigned char charsize;
} PSF1_HEADER;
typedef struct {
PSF1_HEADER* psf1_Header;
void* glyphBuffer;
} PSF1_FONT;
typedef struct {
void* BaseAddress;
size_t BufferSize;
unsigned int Width;
unsigned int Height;
unsigned int PixelsPerScanLine;
} Framebuffer; Framebuffer framebuffer;
Framebuffer* InitializeGOP() {
EFI_GUID gopGuid = EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID;
EFI_GRAPHICS_OUTPUT_PROTOCOL* gop;
EFI_STATUS status;
status = uefi_call_wrapper(BS->LocateProtocol, 3, &gopGuid, NULL, (void**)&gop);
if (EFI_ERROR(status)) {
Print(L"Unable to locate GOP\n\r");
return NULL;
} else {
Print(L"GOP located\n\r");
}
framebuffer.BaseAddress = (void*)gop->Mode->FrameBufferBase;
framebuffer.BufferSize = gop->Mode->FrameBufferSize;
framebuffer.Width = gop->Mode->Info->HorizontalResolution;
framebuffer.Height = gop->Mode->Info->VerticalResolution;
framebuffer.PixelsPerScanLine = gop->Mode->Info->PixelsPerScanLine;
return &framebuffer;
}
EFI_FILE* LoadFile(EFI_FILE* Directory, CHAR16* Path, EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable) {
EFI_FILE* LoadedFile;
EFI_LOADED_IMAGE_PROTOCOL* LoadedImage;
SystemTable->BootServices->HandleProtocol(ImageHandle, &gEfiLoadedImageProtocolGuid, (void**)&LoadedImage);
EFI_SIMPLE_FILE_SYSTEM_PROTOCOL* FileSystem;
SystemTable->BootServices->HandleProtocol(LoadedImage->DeviceHandle, &gEfiSimpleFileSystemProtocolGuid, (void**)&FileSystem);
if (Directory == NULL) {
FileSystem->OpenVolume(FileSystem, &Directory);
}
EFI_STATUS s = Directory->Open(Directory, &LoadedFile, Path, EFI_FILE_MODE_READ, EFI_FILE_READ_ONLY);
if (s != EFI_SUCCESS) {
return NULL;
}
return LoadedFile;
}
PSF1_FONT* LoadPSF1Font(EFI_FILE* Directory, CHAR16* Path, EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable) {
EFI_FILE* font = LoadFile(Directory, Path, ImageHandle, SystemTable);
if (font == NULL) return NULL;
PSF1_HEADER* fontHeader;
SystemTable->BootServices->AllocatePool(EfiLoaderData, sizeof(PSF1_HEADER), (void**)&fontHeader);
UINTN size = sizeof(PSF1_HEADER);
font->Read(font, &size, fontHeader);
if (fontHeader->magic[0] != PSF1_MAGIC0 || fontHeader->magic[1] != PSF1_MAGIC1) return NULL;
UINTN glyphBufferSize = fontHeader->charsize * 256;
if (fontHeader->mode == 1) { // 512 glyph mode
glyphBufferSize *= 2;
}
void* glyphBuffer;
font->SetPosition(font, sizeof(PSF1_HEADER));
SystemTable->BootServices->AllocatePool(EfiLoaderData, glyphBufferSize, (void**)&glyphBuffer);
font->Read(font, &glyphBufferSize, glyphBuffer);
PSF1_FONT* finishedFont;
SystemTable->BootServices->AllocatePool(EfiLoaderData, sizeof(PSF1_FONT), (void**)&finishedFont);
finishedFont->psf1_Header = fontHeader;
finishedFont->glyphBuffer = glyphBuffer;
return finishedFont;
}
int memcmp(const void* aptr, const void* bptr, size_t n) {
const unsigned char* a = aptr, *b = bptr;
for (size_t i = 0; i < n; i++) {
if (a[i] < b[i]) return -1;
else if(a[i] > b[i]) return 1;
}
return 0;
}
EFI_STATUS efi_main (EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE *SystemTable) {
InitializeLib(ImageHandle, SystemTable);
Print(L"Hello World!\n\r");
EFI_FILE* Kernel = LoadFile(NULL, L"kernel.elf", ImageHandle, SystemTable);
if ( Kernel == NULL) {
Print(L"Could not load kernel \n\r");
} else {
Print(L"Kernel Loaded Successfully \n\r");
}
Elf64_Ehdr header;
{
UINTN FileInfoSize;
EFI_FILE_INFO* FileInfo;
Kernel->GetInfo(Kernel, &gEfiFileInfoGuid, &FileInfoSize, NULL);
SystemTable->BootServices->AllocatePool(EfiLoaderData, FileInfoSize, (void**)&FileInfo);
Kernel->GetInfo(Kernel, &gEfiFileInfoGuid, &FileInfoSize, (void**)&FileInfo);
UINTN size = sizeof(header);
Kernel->Read(Kernel, &size, &header);
}
if (
memcmp(&header.e_ident[EI_MAG0], ELFMAG, SELFMAG) != 0 ||
header.e_ident[EI_CLASS] != ELFCLASS64 ||
header.e_ident[EI_DATA] != ELFDATA2LSB ||
header.e_type != ET_EXEC ||
header.e_machine != EM_X86_64 ||
header.e_version != EV_CURRENT
) {
Print(L"kernel format is bad\r\n");
} else {
Print(L"kernel header successfully verified\r\n");
}
Elf64_Phdr* phdrs;
{
Kernel->SetPosition(Kernel, header.e_phoff);
UINTN size = header.e_phnum * header.e_phentsize;
SystemTable->BootServices->AllocatePool(EfiLoaderData, size, (void**)&phdrs);
Kernel->Read(Kernel, &size, phdrs);
}
for (
Elf64_Phdr* phdr = phdrs;
(char*)phdr < (char*)phdrs + header.e_phnum * header.e_phentsize;
phdr = (Elf64_Phdr*)((char*)phdr + header.e_phentsize)
) {
switch(phdr->p_type) {
case PT_LOAD: {
int pages = (phdr->p_memsz + 0x1000 - 1) / 0x1000;
Elf64_Addr segment = phdr->p_paddr;
SystemTable->BootServices->AllocatePages(AllocateAddress, EfiLoaderData, pages, &segment);
Kernel->SetPosition(Kernel, phdr->p_offset);
UINTN size = phdr->p_filesz;
Kernel->Read(Kernel, &size, (void*)segment);
break;
}
}
}
Print(L"Kernel Loaded\n\r");
void (*KernelStart)(Framebuffer*, PSF1_FONT**) = ((__attribute__((sysv_abi)) void(*)(Framebuffer*, PSF1_FONT**) ) header.e_entry);
PSF1_FONT* newFont = LoadPSF1Font(NULL, L"zap-light16.psf", ImageHandle, SystemTable);
if (newFont == NULL) {
Print(L"Font is not valid or is not found\n\r");
} else {
Print(L"Font found, char size = %d\n\r", newFont->psf1_Header->charsize);
}
PSF1_FONT* newFontExt = LoadPSF1Font(NULL, L"zap-ext-light16.psf", ImageHandle, SystemTable);
if (newFont == NULL) {
Print(L"Font is not valid or is not found\n\r");
} else {
Print(L"Font found, char size = %d\n\r", newFont->psf1_Header->charsize);
}
PSF1_FONT* fonts[] = {newFont, newFontExt};
Framebuffer* newBuffer = InitializeGOP();
Print(L"Base: 0x%x\n\rSize: 0x%x\n\rWidth: %d\n\rHeight: %d\n\rPixelsPerScanline: %d\n\r",
newBuffer->BaseAddress,
newBuffer->BufferSize,
newBuffer->Width,
newBuffer->Height,
newBuffer->PixelsPerScanLine);
KernelStart(newBuffer, fonts);
return EFI_SUCCESS; // Exit the UEFI application
}
The problem is here:
if ( (*chr == '\\') && ((*chr+1 == 'n') || (*chr+1 == '0')) ) {
PutChar('\n');
chr++;
chr++;
}
...
You should not be parsing out \n since this will be present in the string as a linefeed character. What you want instead is:
if (*chr == '\n') {
PutChar('\n');
chr++;
}
...

strange behaviour saving file

I use lzmaspec.cpp from 7zip library lzma1900/cpp/7zip/Bundles/Lzmaspec to decompress lzma file.
it works fine and save file correctly.
when I take main2 function and and make header file to use it in my project,it does not save the whole file.
this is lzmaspec.h
/*
* LzmaSpec.h
*
* Created on: Oct 14, 2020
* Author: za2ponubuntu1844
*/
#ifndef LZMASPEC_H_
#define LZMASPEC_H_
int main2(const char *,const char *);
#endif /* LZMASPEC_H_ */
and this is my modified main2 function
/* LzmaSpec.cpp -- LZMA Reference Decoder
2015-06-14 : Igor Pavlov : Public domain */
// This code implements LZMA file decoding according to LZMA specification.
// This code is not optimized for speed.
#include <stdio.h>
#ifdef _MSC_VER
#pragma warning(disable : 4710) // function not inlined
#pragma warning(disable : 4996) // This function or variable may be unsafe
#endif
typedef unsigned char Byte;
typedef unsigned short UInt16;
#ifdef _LZMA_UINT32_IS_ULONG
typedef unsigned long UInt32;
#else
typedef unsigned int UInt32;
#endif
#if defined(_MSC_VER) || defined(__BORLANDC__)
typedef unsigned __int64 UInt64;
#else
typedef unsigned long long int UInt64;
#endif
struct CInputStream
{
FILE *File;
UInt64 Processed;
void Init() { Processed = 0; }
Byte ReadByte()
{
int c = getc(File);
if (c < 0)
throw "Unexpected end of file";
Processed++;
return (Byte)c;
}
};
struct COutStream
{
FILE *File;
UInt64 Processed;
void Init() { Processed = 0; }
void WriteByte(Byte b)
{
if (putc(b, File) == EOF)
throw "File writing error";
Processed++;
}
};
class COutWindow
{
Byte *Buf;
UInt32 Pos;
UInt32 Size;
bool IsFull;
public:
unsigned TotalPos;
COutStream OutStream;
COutWindow(): Buf(NULL) {}
~COutWindow() { delete []Buf; }
void Create(UInt32 dictSize)
{
Buf = new Byte[dictSize];
Pos = 0;
Size = dictSize;
IsFull = false;
TotalPos = 0;
}
void PutByte(Byte b)
{
TotalPos++;
Buf[Pos++] = b;
if (Pos == Size)
{
Pos = 0;
IsFull = true;
}
OutStream.WriteByte(b);
}
Byte GetByte(UInt32 dist) const
{
return Buf[dist <= Pos ? Pos - dist : Size - dist + Pos];
}
void CopyMatch(UInt32 dist, unsigned len)
{
for (; len > 0; len--)
PutByte(GetByte(dist));
}
bool CheckDistance(UInt32 dist) const
{
return dist <= Pos || IsFull;
}
bool IsEmpty() const
{
return Pos == 0 && !IsFull;
}
};
#define kNumBitModelTotalBits 11
#define kNumMoveBits 5
typedef UInt16 CProb;
#define PROB_INIT_VAL ((1 << kNumBitModelTotalBits) / 2)
#define INIT_PROBS(p) \
{ for (unsigned i = 0; i < sizeof(p) / sizeof(p[0]); i++) p[i] = PROB_INIT_VAL; }
class CRangeDecoder
{
UInt32 Range;
UInt32 Code;
void Normalize();
public:
CInputStream *InStream;
bool Corrupted;
bool Init();
bool IsFinishedOK() const { return Code == 0; }
UInt32 DecodeDirectBits(unsigned numBits);
unsigned DecodeBit(CProb *prob);
};
bool CRangeDecoder::Init()
{
Corrupted = false;
Range = 0xFFFFFFFF;
Code = 0;
Byte b = InStream->ReadByte();
for (int i = 0; i < 4; i++)
Code = (Code << 8) | InStream->ReadByte();
if (b != 0 || Code == Range)
Corrupted = true;
return b == 0;
}
#define kTopValue ((UInt32)1 << 24)
void CRangeDecoder::Normalize()
{
if (Range < kTopValue)
{
Range <<= 8;
Code = (Code << 8) | InStream->ReadByte();
}
}
UInt32 CRangeDecoder::DecodeDirectBits(unsigned numBits)
{
UInt32 res = 0;
do
{
Range >>= 1;
Code -= Range;
UInt32 t = 0 - ((UInt32)Code >> 31);
Code += Range & t;
if (Code == Range)
Corrupted = true;
Normalize();
res <<= 1;
res += t + 1;
}
while (--numBits);
return res;
}
unsigned CRangeDecoder::DecodeBit(CProb *prob)
{
unsigned v = *prob;
UInt32 bound = (Range >> kNumBitModelTotalBits) * v;
unsigned symbol;
if (Code < bound)
{
v += ((1 << kNumBitModelTotalBits) - v) >> kNumMoveBits;
Range = bound;
symbol = 0;
}
else
{
v -= v >> kNumMoveBits;
Code -= bound;
Range -= bound;
symbol = 1;
}
*prob = (CProb)v;
Normalize();
return symbol;
}
unsigned BitTreeReverseDecode(CProb *probs, unsigned numBits, CRangeDecoder *rc)
{
unsigned m = 1;
unsigned symbol = 0;
for (unsigned i = 0; i < numBits; i++)
{
unsigned bit = rc->DecodeBit(&probs[m]);
m <<= 1;
m += bit;
symbol |= (bit << i);
}
return symbol;
}
template <unsigned NumBits>
class CBitTreeDecoder
{
CProb Probs[(unsigned)1 << NumBits];
public:
void Init()
{
INIT_PROBS(Probs);
}
unsigned Decode(CRangeDecoder *rc)
{
unsigned m = 1;
for (unsigned i = 0; i < NumBits; i++)
m = (m << 1) + rc->DecodeBit(&Probs[m]);
return m - ((unsigned)1 << NumBits);
}
unsigned ReverseDecode(CRangeDecoder *rc)
{
return BitTreeReverseDecode(Probs, NumBits, rc);
}
};
#define kNumPosBitsMax 4
#define kNumStates 12
#define kNumLenToPosStates 4
#define kNumAlignBits 4
#define kStartPosModelIndex 4
#define kEndPosModelIndex 14
#define kNumFullDistances (1 << (kEndPosModelIndex >> 1))
#define kMatchMinLen 2
class CLenDecoder
{
CProb Choice;
CProb Choice2;
CBitTreeDecoder<3> LowCoder[1 << kNumPosBitsMax];
CBitTreeDecoder<3> MidCoder[1 << kNumPosBitsMax];
CBitTreeDecoder<8> HighCoder;
public:
void Init()
{
Choice = PROB_INIT_VAL;
Choice2 = PROB_INIT_VAL;
HighCoder.Init();
for (unsigned i = 0; i < (1 << kNumPosBitsMax); i++)
{
LowCoder[i].Init();
MidCoder[i].Init();
}
}
unsigned Decode(CRangeDecoder *rc, unsigned posState)
{
if (rc->DecodeBit(&Choice) == 0)
return LowCoder[posState].Decode(rc);
if (rc->DecodeBit(&Choice2) == 0)
return 8 + MidCoder[posState].Decode(rc);
return 16 + HighCoder.Decode(rc);
}
};
unsigned UpdateState_Literal(unsigned state)
{
if (state < 4) return 0;
else if (state < 10) return state - 3;
else return state - 6;
}
unsigned UpdateState_Match (unsigned state) { return state < 7 ? 7 : 10; }
unsigned UpdateState_Rep (unsigned state) { return state < 7 ? 8 : 11; }
unsigned UpdateState_ShortRep(unsigned state) { return state < 7 ? 9 : 11; }
#define LZMA_DIC_MIN (1 << 12)
class CLzmaDecoder
{
public:
CRangeDecoder RangeDec;
COutWindow OutWindow;
bool markerIsMandatory;
unsigned lc, pb, lp;
UInt32 dictSize;
UInt32 dictSizeInProperties;
void DecodeProperties(const Byte *properties)
{
unsigned d = properties[0];
if (d >= (9 * 5 * 5))
throw "Incorrect LZMA properties";
lc = d % 9;
d /= 9;
pb = d / 5;
lp = d % 5;
dictSizeInProperties = 0;
for (int i = 0; i < 4; i++)
dictSizeInProperties |= (UInt32)properties[i + 1] << (8 * i);
dictSize = dictSizeInProperties;
if (dictSize < LZMA_DIC_MIN)
dictSize = LZMA_DIC_MIN;
}
CLzmaDecoder(): LitProbs(NULL) {}
~CLzmaDecoder() { delete []LitProbs; }
void Create()
{
OutWindow.Create(dictSize);
CreateLiterals();
}
int Decode(bool unpackSizeDefined, UInt64 unpackSize);
private:
CProb *LitProbs;
void CreateLiterals()
{
LitProbs = new CProb[(UInt32)0x300 << (lc + lp)];
}
void InitLiterals()
{
UInt32 num = (UInt32)0x300 << (lc + lp);
for (UInt32 i = 0; i < num; i++)
LitProbs[i] = PROB_INIT_VAL;
}
void DecodeLiteral(unsigned state, UInt32 rep0)
{
unsigned prevByte = 0;
if (!OutWindow.IsEmpty())
prevByte = OutWindow.GetByte(1);
unsigned symbol = 1;
unsigned litState = ((OutWindow.TotalPos & ((1 << lp) - 1)) << lc) + (prevByte >> (8 - lc));
CProb *probs = &LitProbs[(UInt32)0x300 * litState];
if (state >= 7)
{
unsigned matchByte = OutWindow.GetByte(rep0 + 1);
do
{
unsigned matchBit = (matchByte >> 7) & 1;
matchByte <<= 1;
unsigned bit = RangeDec.DecodeBit(&probs[((1 + matchBit) << 8) + symbol]);
symbol = (symbol << 1) | bit;
if (matchBit != bit)
break;
}
while (symbol < 0x100);
}
while (symbol < 0x100)
symbol = (symbol << 1) | RangeDec.DecodeBit(&probs[symbol]);
OutWindow.PutByte((Byte)(symbol - 0x100));
}
CBitTreeDecoder<6> PosSlotDecoder[kNumLenToPosStates];
CBitTreeDecoder<kNumAlignBits> AlignDecoder;
CProb PosDecoders[1 + kNumFullDistances - kEndPosModelIndex];
void InitDist()
{
for (unsigned i = 0; i < kNumLenToPosStates; i++)
PosSlotDecoder[i].Init();
AlignDecoder.Init();
INIT_PROBS(PosDecoders);
}
unsigned DecodeDistance(unsigned len)
{
unsigned lenState = len;
if (lenState > kNumLenToPosStates - 1)
lenState = kNumLenToPosStates - 1;
unsigned posSlot = PosSlotDecoder[lenState].Decode(&RangeDec);
if (posSlot < 4)
return posSlot;
unsigned numDirectBits = (unsigned)((posSlot >> 1) - 1);
UInt32 dist = ((2 | (posSlot & 1)) << numDirectBits);
if (posSlot < kEndPosModelIndex)
dist += BitTreeReverseDecode(PosDecoders + dist - posSlot, numDirectBits, &RangeDec);
else
{
dist += RangeDec.DecodeDirectBits(numDirectBits - kNumAlignBits) << kNumAlignBits;
dist += AlignDecoder.ReverseDecode(&RangeDec);
}
return dist;
}
CProb IsMatch[kNumStates << kNumPosBitsMax];
CProb IsRep[kNumStates];
CProb IsRepG0[kNumStates];
CProb IsRepG1[kNumStates];
CProb IsRepG2[kNumStates];
CProb IsRep0Long[kNumStates << kNumPosBitsMax];
CLenDecoder LenDecoder;
CLenDecoder RepLenDecoder;
void Init()
{
InitLiterals();
InitDist();
INIT_PROBS(IsMatch);
INIT_PROBS(IsRep);
INIT_PROBS(IsRepG0);
INIT_PROBS(IsRepG1);
INIT_PROBS(IsRepG2);
INIT_PROBS(IsRep0Long);
LenDecoder.Init();
RepLenDecoder.Init();
}
};
#define LZMA_RES_ERROR 0
#define LZMA_RES_FINISHED_WITH_MARKER 1
#define LZMA_RES_FINISHED_WITHOUT_MARKER 2
int CLzmaDecoder::Decode(bool unpackSizeDefined, UInt64 unpackSize)
{
if (!RangeDec.Init())
return LZMA_RES_ERROR;
Init();
UInt32 rep0 = 0, rep1 = 0, rep2 = 0, rep3 = 0;
unsigned state = 0;
for (;;)
{
if (unpackSizeDefined && unpackSize == 0 && !markerIsMandatory)
if (RangeDec.IsFinishedOK())
return LZMA_RES_FINISHED_WITHOUT_MARKER;
unsigned posState = OutWindow.TotalPos & ((1 << pb) - 1);
if (RangeDec.DecodeBit(&IsMatch[(state << kNumPosBitsMax) + posState]) == 0)
{
if (unpackSizeDefined && unpackSize == 0)
return LZMA_RES_ERROR;
DecodeLiteral(state, rep0);
state = UpdateState_Literal(state);
unpackSize--;
continue;
}
unsigned len;
if (RangeDec.DecodeBit(&IsRep[state]) != 0)
{
if (unpackSizeDefined && unpackSize == 0)
return LZMA_RES_ERROR;
if (OutWindow.IsEmpty())
return LZMA_RES_ERROR;
if (RangeDec.DecodeBit(&IsRepG0[state]) == 0)
{
if (RangeDec.DecodeBit(&IsRep0Long[(state << kNumPosBitsMax) + posState]) == 0)
{
state = UpdateState_ShortRep(state);
OutWindow.PutByte(OutWindow.GetByte(rep0 + 1));
unpackSize--;
continue;
}
}
else
{
UInt32 dist;
if (RangeDec.DecodeBit(&IsRepG1[state]) == 0)
dist = rep1;
else
{
if (RangeDec.DecodeBit(&IsRepG2[state]) == 0)
dist = rep2;
else
{
dist = rep3;
rep3 = rep2;
}
rep2 = rep1;
}
rep1 = rep0;
rep0 = dist;
}
len = RepLenDecoder.Decode(&RangeDec, posState);
state = UpdateState_Rep(state);
}
else
{
rep3 = rep2;
rep2 = rep1;
rep1 = rep0;
len = LenDecoder.Decode(&RangeDec, posState);
state = UpdateState_Match(state);
rep0 = DecodeDistance(len);
if (rep0 == 0xFFFFFFFF)
return RangeDec.IsFinishedOK() ?
LZMA_RES_FINISHED_WITH_MARKER :
LZMA_RES_ERROR;
if (unpackSizeDefined && unpackSize == 0)
return LZMA_RES_ERROR;
if (rep0 >= dictSize || !OutWindow.CheckDistance(rep0))
return LZMA_RES_ERROR;
}
len += kMatchMinLen;
bool isError = false;
if (unpackSizeDefined && unpackSize < len)
{
len = (unsigned)unpackSize;
isError = true;
}
OutWindow.CopyMatch(rep0 + 1, len);
unpackSize -= len;
if (isError)
return LZMA_RES_ERROR;
}
}
static void Print(const char *s)
{
fputs(s, stdout);
}
static void PrintError(const char *s)
{
fputs(s, stderr);
}
#define CONVERT_INT_TO_STR(charType, tempSize) \
void ConvertUInt64ToString(UInt64 val, char *s)
{
char temp[32];
unsigned i = 0;
while (val >= 10)
{
temp[i++] = (char)('0' + (unsigned)(val % 10));
val /= 10;
}
*s++ = (char)('0' + (unsigned)val);
while (i != 0)
{
i--;
*s++ = temp[i];
}
*s = 0;
}
void PrintUInt64(const char *title, UInt64 v)
{
Print(title);
Print(" : ");
char s[32];
ConvertUInt64ToString(v, s);
Print(s);
Print(" bytes \n");
}
int main2(const char * infile, const char *outfile)
{
try
{
Print("\nLZMA Reference Decoder 15.00 : Igor Pavlov : Public domain : 2015-04-16\n");
//if (numArgs == 1)
// Print("\nUse: lzmaSpec a.lzma outFile");
//if (numArgs != 3)
// throw "you must specify two parameters";
CInputStream inStream;
inStream.File = fopen(infile, "rb");
inStream.Init();
if (inStream.File == 0)
throw "Can't open input file";
CLzmaDecoder lzmaDecoder;
lzmaDecoder.OutWindow.OutStream.File = fopen(outfile, "wb+");
lzmaDecoder.OutWindow.OutStream.Init();
if (inStream.File == 0)
throw "Can't open output file";
Byte header[13];
int i;
for (i = 0; i < 13; i++)
header[i] = inStream.ReadByte();
lzmaDecoder.DecodeProperties(header);
printf("\nlc=%d, lp=%d, pb=%d", lzmaDecoder.lc, lzmaDecoder.lp, lzmaDecoder.pb);
printf("\nDictionary Size in properties = %u", lzmaDecoder.dictSizeInProperties);
printf("\nDictionary Size for decoding = %u", lzmaDecoder.dictSize);
UInt64 unpackSize = 0;
bool unpackSizeDefined = false;
for (i = 0; i < 8; i++)
{
Byte b = header[5 + i];
if (b != 0xFF)
unpackSizeDefined = true;
unpackSize |= (UInt64)b << (8 * i);
}
lzmaDecoder.markerIsMandatory = !unpackSizeDefined;
Print("\n");
if (unpackSizeDefined)
PrintUInt64("Uncompressed Size", unpackSize);
else
Print("End marker is expected\n");
lzmaDecoder.RangeDec.InStream = &inStream;
Print("\n");
lzmaDecoder.Create();
int res = lzmaDecoder.Decode(unpackSizeDefined, unpackSize);
PrintUInt64("Read ", inStream.Processed);
PrintUInt64("Written ", lzmaDecoder.OutWindow.OutStream.Processed);
if (res == LZMA_RES_ERROR)
throw "LZMA decoding error";
else if (res == LZMA_RES_FINISHED_WITHOUT_MARKER)
Print("Finished without end marker");
else if (res == LZMA_RES_FINISHED_WITH_MARKER)
{
if (unpackSizeDefined)
{
if (lzmaDecoder.OutWindow.OutStream.Processed != unpackSize)
throw "Finished with end marker before than specified size";
Print("Warning: ");
}
Print("Finished with end marker");
}
else
throw "Internal Error";
Print("\n");
if (lzmaDecoder.RangeDec.Corrupted)
{
Print("\nWarning: LZMA stream is corrupted\n");
}
//15-10-2020
}
catch (const char *s)
{
PrintError("\nError:\n");
PrintError(s);
PrintError("\n");
return 1;
}
catch(...)
{
PrintError("\nError\n");
return 1;
}
return 0;
}
/*
int
#ifdef _MSC_VER
__cdecl
#endif
main(int numArgs, const char *args[])
{
try { return main2(numArgs, args); }
catch (const char *s)
{
PrintError("\nError:\n");
PrintError(s);
PrintError("\n");
return 1;
}
catch(...)
{
PrintError("\nError\n");
return 1;
}
}
*/
when i debug it i find that function
void WriteByte(Byte b)
works fine but the resulting file does not have correct size.
i used this file
13h_ticks.bi5 from dukascopy
i uploaded it here
https://github.com/ahmed-allam/starting_coding
I do not know why this happens.
when i try to decompress same file using lzmaspec.cpp outside project ,it works fine.
I need guide line where the error could be?

How to asynchronously read/write in C++?

How do you copy one stream to another using dedicated read/write threads in C++?
Let's say I have these methods (not real, but to illustrate the point) to read/write data from. These read/write functions could represent anything (network/file/USB/serial/etc).
// returns the number of bytes read
void read(char* buffer, int bufferSize, int* bytesRead);
// returns the number of bytes written
void write(char* buffer, int bufferSize, int* bytesWritten);
The solution should also be portable.
NOTE: I am aware that Windows has a FILE_FLAG_OVERLAPPED feature, but this assumes that the read/write is file IO. Remember, these read/write methods could represent anything.
Here is the solution I came up with.
Header
#pragma once
#include <stdlib.h>
#include <queue>
#include <mutex>
#include <thread>
#include <chrono>
#include <list>
#include <thread>
#define ASYNC_COPY_READ_WRITE_SUCCESS 0
struct BufferBlock;
struct ReadStream
{
// read a stream to a buffer.
// return non-zero if error occured
virtual int read(char* buffer, int bufferSize, int* bytesRead) = 0;
};
struct WriteStream
{
// write a buffer to a stream.
// return non-zero if error occured
virtual int write(char* buffer, int bufferSize, int* bytesWritten) = 0;
};
class BufferBlockManager
{
public:
BufferBlockManager(int numberOfBlocks, int bufferSize);
~BufferBlockManager();
void enqueueBlockForRead(BufferBlock* block);
void dequeueBlockForRead(BufferBlock** block);
void enqueueBlockForWrite(BufferBlock* block);
void dequeueBlockForWrite(BufferBlock** block);
void resetState();
private:
std::list<BufferBlock*> blocks;
std::queue<BufferBlock*> blocksPendingRead;
std::queue<BufferBlock*> blocksPendingWrite;
std::mutex queueLock;
std::chrono::milliseconds dequeueSleepTime;
};
void AsyncCopyStream(BufferBlockManager* bufferBlockManager, ReadStream* readStream, WriteStream* writeStream, int* readResult, int* writeResult);
CPP
#include "AsyncReadWrite.h"
struct BufferBlock
{
BufferBlock(int bufferSize) : buffer(NULL)
{
this->bufferSize = bufferSize;
this->buffer = new char[bufferSize];
this->actualSize = 0;
this->isLastBlock = false;
}
~BufferBlock()
{
this->bufferSize = 0;
free(this->buffer);
this->buffer = NULL;
this->actualSize = 0;
}
char* buffer;
int bufferSize;
int actualSize;
bool isLastBlock;
};
BufferBlockManager::BufferBlockManager(int numberOfBlocks, int bufferSize)
{
dequeueSleepTime = std::chrono::milliseconds(100);
for (int x = 0; x < numberOfBlocks; x++)
{
BufferBlock* block = new BufferBlock(bufferSize);
blocks.push_front(block);
blocksPendingRead.push(block);
}
}
BufferBlockManager::~BufferBlockManager()
{
for (std::list<BufferBlock*>::const_iterator iterator = blocks.begin(), end = blocks.end(); iterator != end; ++iterator) {
delete (*iterator);
}
}
void BufferBlockManager::enqueueBlockForRead(BufferBlock* block)
{
queueLock.lock();
block->actualSize = 0;
block->isLastBlock = false;
blocksPendingRead.push(block);
queueLock.unlock();
}
void BufferBlockManager::dequeueBlockForRead(BufferBlock** block)
{
WAITFOR:
while (blocksPendingRead.size() == 0)
std::this_thread::sleep_for(dequeueSleepTime);
queueLock.lock();
if (blocksPendingRead.size() == 0)
{
queueLock.unlock();
goto WAITFOR;
}
*block = blocksPendingRead.front();
blocksPendingRead.pop();
queueLock.unlock();
}
void BufferBlockManager::enqueueBlockForWrite(BufferBlock* block)
{
queueLock.lock();
blocksPendingWrite.push(block);
queueLock.unlock();
}
void BufferBlockManager::dequeueBlockForWrite(BufferBlock** block)
{
WAITFOR:
while (blocksPendingWrite.size() == 0)
std::this_thread::sleep_for(dequeueSleepTime);
queueLock.lock();
if (blocksPendingWrite.size() == 0)
{
queueLock.unlock();
goto WAITFOR;
}
*block = blocksPendingWrite.front();
blocksPendingWrite.pop();
queueLock.unlock();
}
void BufferBlockManager::resetState()
{
queueLock.lock();
blocksPendingRead = std::queue<BufferBlock*>();
blocksPendingWrite = std::queue<BufferBlock*>();
for (std::list<BufferBlock*>::const_iterator iterator = blocks.begin(), end = blocks.end(); iterator != end; ++iterator) {
(*iterator)->actualSize = 0;
}
queueLock.unlock();
}
struct AsyncCopyContext
{
AsyncCopyContext(BufferBlockManager* bufferBlockManager, ReadStream* readStream, WriteStream* writeStream)
{
this->bufferBlockManager = bufferBlockManager;
this->readStream = readStream;
this->writeStream = writeStream;
this->readResult = ASYNC_COPY_READ_WRITE_SUCCESS;
this->writeResult = ASYNC_COPY_READ_WRITE_SUCCESS;
}
BufferBlockManager* bufferBlockManager;
ReadStream* readStream;
WriteStream* writeStream;
int readResult;
int writeResult;
};
void ReadStreamThread(AsyncCopyContext* asyncContext)
{
int bytesRead = 0;
BufferBlock* readBuffer = NULL;
int readResult = ASYNC_COPY_READ_WRITE_SUCCESS;
while (
// as long there hasn't been any write errors
asyncContext->writeResult == ASYNC_COPY_READ_WRITE_SUCCESS
// and we haven't had an error reading yet
&& readResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
// let's deque a block to read to!
asyncContext->bufferBlockManager->dequeueBlockForRead(&readBuffer);
readResult = asyncContext->readStream->read(readBuffer->buffer, readBuffer->bufferSize, &bytesRead);
readBuffer->actualSize = bytesRead;
readBuffer->isLastBlock = bytesRead == 0;
if (readResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
// this was a valid read, go ahead and queue it for writing
asyncContext->bufferBlockManager->enqueueBlockForWrite(readBuffer);
}
else
{
// an error occured reading
asyncContext->readResult = readResult;
// since an error occured, lets queue an block to write indicatiting we are done and there are no more bytes to read
readBuffer->isLastBlock = true;
readBuffer->actualSize = 0;
asyncContext->bufferBlockManager->enqueueBlockForWrite(readBuffer);
}
if (readBuffer->isLastBlock) return;
}
}
void WriteStreamThread(AsyncCopyContext* asyncContext)
{
int bytesWritten = 0;
BufferBlock* writeBuffer = NULL;
int writeResult = ASYNC_COPY_READ_WRITE_SUCCESS;
bool isLastWriteBlock = false;
while (
// as long as there are no errors during reading
asyncContext->readResult == ASYNC_COPY_READ_WRITE_SUCCESS
// and we haven't had an error writing yet
&& writeResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
// lets dequeue a block for writing!
asyncContext->bufferBlockManager->dequeueBlockForWrite(&writeBuffer);
isLastWriteBlock = writeBuffer->isLastBlock;
if (writeBuffer->actualSize > 0)
writeResult = asyncContext->writeStream->write(writeBuffer->buffer, writeBuffer->actualSize, &bytesWritten);
if (writeResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
asyncContext->bufferBlockManager->enqueueBlockForRead(writeBuffer);
if (isLastWriteBlock) return;
}
else
{
asyncContext->writeResult = writeResult;
asyncContext->bufferBlockManager->enqueueBlockForRead(writeBuffer);
return;
}
}
}
void AsyncCopyStream(BufferBlockManager* bufferBlockManager, ReadStream* readStream, WriteStream* writeStream, int* readResult, int* writeResult)
{
AsyncCopyContext asyncContext(bufferBlockManager, readStream, writeStream);
std::thread readThread(ReadStreamThread, &asyncContext);
std::thread writeThread(WriteStreamThread, &asyncContext);
readThread.join();
writeThread.join();
*readResult = asyncContext.readResult;
*writeResult = asyncContext.writeResult;
}
Usage
#include <stdio.h>
#include <tchar.h>
#include "AsyncReadWrite.h"
struct ReadTestStream : ReadStream
{
int readCount = 0;
int read(char* buffer, int bufferSize, int* bytesRead)
{
printf("Starting read...\n");
memset(buffer, bufferSize, 0);
if (readCount == 10)
{
*bytesRead = 0;
return 0;
}
// pretend this function takes a while!
std::this_thread::sleep_for(std::chrono::milliseconds(100));
char buff[100];
sprintf_s(buff, "This is read number %d\n", readCount);
strcpy_s(buffer, sizeof(buff), buff);
*bytesRead = strlen(buffer);
readCount++;
printf("Finished read...\n");
return 0;
}
};
struct WriteTestStream : WriteStream
{
int write(char* buffer, int bufferSize, int* bytesWritten)
{
printf("Starting write...\n");
// pretend this function takes a while!
std::this_thread::sleep_for(std::chrono::milliseconds(500));
printf(buffer);
printf("Finished write...\n");
return 0;
}
};
int _tmain(int argc, _TCHAR* argv[])
{
BufferBlockManager bufferBlockManager(5, 4096);
ReadTestStream readStream;
WriteTestStream writeStream;
int readResult = 0;
int writeResult = 0;
printf("Starting copy...\n");
AsyncCopyStream(&bufferBlockManager, &readStream, &writeStream, &readResult, &writeResult);
printf("Finished copy... readResult=%d writeResult=%d \n", readResult, writeResult);
getchar();
return 0;
}
EDIT: I put my solution into a GitHub repository here. If you wish to use this code, refer to the repository since it may be more updated than this answer.
Typically, you would just have one thread for each direction that alternates between reads and writes.

How to use LZMA SDK in C++?

i have difficulties in using LZMA SDK in my application.
I would like to create a kind of single file compression tool. I dont need any directory support, just need only the LZMA2 stream. But i have no idea on how LZMA SDK is to be used for this.
Please can anyone give me a little example on how the LZMA SDK can be used under C++?
I think that it's a properly little example to use LZMA SDK.
/* LzmaUtil.c -- Test application for LZMA compression
2008-08-05
Igor Pavlov
public domain */
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../LzmaDec.h"
#include "../LzmaEnc.h"
#include "../Alloc.h"
const char *kCantReadMessage = "Can not read input file";
const char *kCantWriteMessage = "Can not write output file";
const char *kCantAllocateMessage = "Can not allocate memory";
const char *kDataErrorMessage = "Data error";
static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
static void SzFree(void *p, void *address) { p = p; MyFree(address); }
static ISzAlloc g_Alloc = { SzAlloc, SzFree };
#define kInBufferSize (1 << 15)
#define kOutBufferSize (1 << 15)
unsigned char g_InBuffer[kInBufferSize];
unsigned char g_OutBuffer[kOutBufferSize];
size_t MyReadFile(FILE *file, void *data, size_t size)
{ return fread(data, 1, size, file); }
int MyReadFileAndCheck(FILE *file, void *data, size_t size)
{ return (MyReadFile(file, data, size) == size); }
size_t MyWriteFile(FILE *file, const void *data, size_t size)
{
if (size == 0)
return 0;
return fwrite(data, 1, size, file);
}
int MyWriteFileAndCheck(FILE *file, const void *data, size_t size)
{ return (MyWriteFile(file, data, size) == size); }
long MyGetFileLength(FILE *file)
{
long length;
fseek(file, 0, SEEK_END);
length = ftell(file);
fseek(file, 0, SEEK_SET);
return length;
}
void PrintHelp(char *buffer)
{
strcat(buffer, "\nLZMA Utility 4.58 Copyright (c) 1999-2008 Igor Pavlov 2008-04-11\n"
"\nUsage: lzma <e|d> inputFile outputFile\n"
" e: encode file\n"
" d: decode file\n");
}
int PrintError(char *buffer, const char *message)
{
strcat(buffer, "\nError: ");
strcat(buffer, message);
strcat(buffer, "\n");
return 1;
}
int PrintErrorNumber(char *buffer, SRes val)
{
sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val);
return 1;
}
int PrintUserError(char *buffer)
{
return PrintError(buffer, "Incorrect command");
}
#define IN_BUF_SIZE (1 << 16)
#define OUT_BUF_SIZE (1 << 16)
static int Decode(FILE *inFile, FILE *outFile, char *rs)
{
UInt64 unpackSize;
int thereIsSize; /* = 1, if there is uncompressed size in headers */
int i;
int res = 0;
CLzmaDec state;
/* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */
unsigned char header[LZMA_PROPS_SIZE + 8];
/* Read and parse header */
if (!MyReadFileAndCheck(inFile, header, sizeof(header)))
return PrintError(rs, kCantReadMessage);
unpackSize = 0;
thereIsSize = 0;
for (i = 0; i < 8; i++)
{
unsigned char b = header[LZMA_PROPS_SIZE + i];
if (b != 0xFF)
thereIsSize = 1;
unpackSize += (UInt64)b << (i * 8);
}
LzmaDec_Construct(&state);
res = LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc);
if (res != SZ_OK)
return res;
{
Byte inBuf[IN_BUF_SIZE];
Byte outBuf[OUT_BUF_SIZE];
size_t inPos = 0, inSize = 0, outPos = 0;
LzmaDec_Init(&state);
for (;;)
{
if (inPos == inSize)
{
inSize = MyReadFile(inFile, inBuf, IN_BUF_SIZE);
inPos = 0;
}
{
SizeT inProcessed = inSize - inPos;
SizeT outProcessed = OUT_BUF_SIZE - outPos;
ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
ELzmaStatus status;
if (thereIsSize && outProcessed > unpackSize)
{
outProcessed = (SizeT)unpackSize;
finishMode = LZMA_FINISH_END;
}
res = LzmaDec_DecodeToBuf(&state, outBuf + outPos, &outProcessed,
inBuf + inPos, &inProcessed, finishMode, &status);
inPos += (UInt32)inProcessed;
outPos += outProcessed;
unpackSize -= outProcessed;
if (outFile != 0)
MyWriteFile(outFile, outBuf, outPos);
outPos = 0;
if (res != SZ_OK || thereIsSize && unpackSize == 0)
break;
if (inProcessed == 0 && outProcessed == 0)
{
if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK)
res = SZ_ERROR_DATA;
break;
}
}
}
}
LzmaDec_Free(&state, &g_Alloc);
return res;
}
typedef struct _CFileSeqInStream
{
ISeqInStream funcTable;
FILE *file;
} CFileSeqInStream;
static SRes MyRead(void *p, void *buf, size_t *size)
{
if (*size == 0)
return SZ_OK;
*size = MyReadFile(((CFileSeqInStream*)p)->file, buf, *size);
/*
if (*size == 0)
return SZE_FAIL;
*/
return SZ_OK;
}
typedef struct _CFileSeqOutStream
{
ISeqOutStream funcTable;
FILE *file;
} CFileSeqOutStream;
static size_t MyWrite(void *pp, const void *buf, size_t size)
{
return MyWriteFile(((CFileSeqOutStream *)pp)->file, buf, size);
}
static SRes Encode(FILE *inFile, FILE *outFile, char *rs)
{
CLzmaEncHandle enc;
SRes res;
CFileSeqInStream inStream;
CFileSeqOutStream outStream;
CLzmaEncProps props;
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
return SZ_ERROR_MEM;
inStream.funcTable.Read = MyRead;
inStream.file = inFile;
outStream.funcTable.Write = MyWrite;
outStream.file = outFile;
LzmaEncProps_Init(&props);
res = LzmaEnc_SetProps(enc, &props);
if (res == SZ_OK)
{
Byte header[LZMA_PROPS_SIZE + 8];
size_t headerSize = LZMA_PROPS_SIZE;
UInt64 fileSize;
int i;
res = LzmaEnc_WriteProperties(enc, header, &headerSize);
fileSize = MyGetFileLength(inFile);
for (i = 0; i < 8; i++)
header[headerSize++] = (Byte)(fileSize >> (8 * i));
if (!MyWriteFileAndCheck(outFile, header, headerSize))
return PrintError(rs, "writing error");
if (res == SZ_OK)
res = LzmaEnc_Encode(enc, &outStream.funcTable, &inStream.funcTable,
NULL, &g_Alloc, &g_Alloc);
}
LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
return res;
}
int main2(int numArgs, const char *args[], char *rs)
{
FILE *inFile = 0;
FILE *outFile = 0;
char c;
int res;
int encodeMode;
if (numArgs == 1)
{
PrintHelp(rs);
return 0;
}
if (numArgs < 3 || numArgs > 4 || strlen(args[1]) != 1)
return PrintUserError(rs);
c = args[1][0];
encodeMode = (c == 'e' || c == 'E');
if (!encodeMode && c != 'd' && c != 'D')
return PrintUserError(rs);
{
size_t t4 = sizeof(UInt32);
size_t t8 = sizeof(UInt64);
if (t4 != 4 || t8 != 8)
return PrintError(rs, "LZMA UTil needs correct UInt32 and UInt64");
}
inFile = fopen(args[2], "rb");
if (inFile == 0)
return PrintError(rs, "Can not open input file");
if (numArgs > 3)
{
outFile = fopen(args[3], "wb+");
if (outFile == 0)
return PrintError(rs, "Can not open output file");
}
else if (encodeMode)
PrintUserError(rs);
if (encodeMode)
{
res = Encode(inFile, outFile, rs);
}
else
{
res = Decode(inFile, outFile, rs);
}
if (outFile != 0)
fclose(outFile);
fclose(inFile);
if (res != SZ_OK)
{
if (res == SZ_ERROR_MEM)
return PrintError(rs, kCantAllocateMessage);
else if (res == SZ_ERROR_DATA)
return PrintError(rs, kDataErrorMessage);
else
return PrintErrorNumber(rs, res);
}
return 0;
}
int MY_CDECL main(int numArgs, const char *args[])
{
char rs[800] = { 0 };
int res = main2(numArgs, args, rs);
printf(rs);
return res;
}
Also you can see it at:
http://read.pudn.com/downloads151/sourcecode/zip/656407/7z460/C/LzmaUtil/LzmaUtil.c__.htm
http://read.pudn.com/downloads157/sourcecode/zip/698262/LZMA/LzmaUtil.c__.htm
I recently found a nice example, written in C++. Credit goes to GH user Treeki who published the original gist:
// note: -D_7ZIP_ST is required when compiling on non-Windows platforms
// g++ -o lzma_sample -std=c++14 -D_7ZIP_ST lzma_sample.cpp LzmaDec.c LzmaEnc.c LzFind.c
#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <memory>
#include "LzmaEnc.h"
#include "LzmaDec.h"
static void *_lzmaAlloc(ISzAllocPtr, size_t size) {
return new uint8_t[size];
}
static void _lzmaFree(ISzAllocPtr, void *addr) {
if (!addr)
return;
delete[] reinterpret_cast<uint8_t *>(addr);
}
static ISzAlloc _allocFuncs = {
_lzmaAlloc, _lzmaFree
};
std::unique_ptr<uint8_t[]> lzmaCompress(const uint8_t *input, uint32_t inputSize, uint32_t *outputSize) {
std::unique_ptr<uint8_t[]> result;
// set up properties
CLzmaEncProps props;
LzmaEncProps_Init(&props);
if (inputSize >= (1 << 20))
props.dictSize = 1 << 20; // 1mb dictionary
else
props.dictSize = inputSize; // smaller dictionary = faster!
props.fb = 40;
// prepare space for the encoded properties
SizeT propsSize = 5;
uint8_t propsEncoded[5];
// allocate some space for the compression output
// this is way more than necessary in most cases...
// but better safe than sorry
// (a smarter implementation would use a growing buffer,
// but this requires a bunch of fuckery that is out of
/// scope for this simple example)
SizeT outputSize64 = inputSize * 1.5;
if (outputSize64 < 1024)
outputSize64 = 1024;
auto output = std::make_unique<uint8_t[]>(outputSize64);
int lzmaStatus = LzmaEncode(
output.get(), &outputSize64, input, inputSize,
&props, propsEncoded, &propsSize, 0,
NULL,
&_allocFuncs, &_allocFuncs);
*outputSize = outputSize64 + 13;
if (lzmaStatus == SZ_OK) {
// tricky: we have to generate the LZMA header
// 5 bytes properties + 8 byte uncompressed size
result = std::make_unique<uint8_t[]>(outputSize64 + 13);
uint8_t *resultData = result.get();
memcpy(resultData, propsEncoded, 5);
for (int i = 0; i < 8; i++)
resultData[5 + i] = (inputSize >> (i * 8)) & 0xFF;
memcpy(resultData + 13, output.get(), outputSize64);
}
return result;
}
std::unique_ptr<uint8_t[]> lzmaDecompress(const uint8_t *input, uint32_t inputSize, uint32_t *outputSize) {
if (inputSize < 13)
return NULL; // invalid header!
// extract the size from the header
UInt64 size = 0;
for (int i = 0; i < 8; i++)
size |= (input[5 + i] << (i * 8));
if (size <= (256 * 1024 * 1024)) {
auto blob = std::make_unique<uint8_t[]>(size);
ELzmaStatus lzmaStatus;
SizeT procOutSize = size, procInSize = inputSize - 13;
int status = LzmaDecode(blob.get(), &procOutSize, &input[13], &procInSize, input, 5, LZMA_FINISH_END, &lzmaStatus, &_allocFuncs);
if (status == SZ_OK && procOutSize == size) {
*outputSize = size;
return blob;
}
}
return NULL;
}
void hexdump(const uint8_t *buf, int size) {
int lines = (size + 15) / 16;
for (int i = 0; i < lines; i++) {
printf("%08x | ", i * 16);
int lineMin = i * 16;
int lineMax = lineMin + 16;
int lineCappedMax = (lineMax > size) ? size : lineMax;
for (int j = lineMin; j < lineCappedMax; j++)
printf("%02x ", buf[j]);
for (int j = lineCappedMax; j < lineMax; j++)
printf(" ");
printf("| ");
for (int j = lineMin; j < lineCappedMax; j++) {
if (buf[j] >= 32 && buf[j] <= 127)
printf("%c", buf[j]);
else
printf(".");
}
printf("\n");
}
}
void testIt(const uint8_t *input, int size) {
printf("Test Input:\n");
hexdump(input, size);
uint32_t compressedSize;
auto compressedBlob = lzmaCompress(input, size, &compressedSize);
if (compressedBlob) {
printf("Compressed:\n");
hexdump(compressedBlob.get(), compressedSize);
} else {
printf("Nope, we screwed it\n");
return;
}
// let's try decompressing it now
uint32_t decompressedSize;
auto decompressedBlob = lzmaDecompress(compressedBlob.get(), compressedSize, &decompressedSize);
if (decompressedBlob) {
printf("Decompressed:\n");
hexdump(decompressedBlob.get(), decompressedSize);
} else {
printf("Nope, we screwed it (part 2)\n");
return;
}
printf("----------\n");
}
void testIt(const char *string) {
testIt((const uint8_t *)string, strlen(string));
}
int main(int argc, char **argv) {
testIt("a");
testIt("here is a cool string");
testIt("here's something that should compress pretty well: abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdef");
return 0;
}
You can refer to this file on how to use lzma2。
https://github.com/Tencent/libpag/blob/aab6391e455193c8ec5b8e2031b495b3fe77b034/test/framework/utils/LzmaUtil.cpp
/////////////////////////////////////////////////////////////////////////////////////////////////
//
// Tencent is pleased to support the open source community by making libpag available.
//
// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// unless required by applicable law or agreed to in writing, software distributed under the
// license is distributed on an "as is" basis, without warranties or conditions of any kind,
// either express or implied. see the license for the specific language governing permissions
// and limitations under the license.
//
/////////////////////////////////////////////////////////////////////////////////////////////////
#include "LzmaUtil.h"
#include "test/framework/lzma/Lzma2DecMt.h"
#include "test/framework/lzma/Lzma2Enc.h"
namespace pag {
static void* LzmaAlloc(ISzAllocPtr, size_t size) {
return new uint8_t[size];
}
static void LzmaFree(ISzAllocPtr, void* address) {
if (!address) {
return;
}
delete[] reinterpret_cast<uint8_t*>(address);
}
static ISzAlloc gAllocFuncs = {LzmaAlloc, LzmaFree};
class SequentialOutStream {
public:
virtual ~SequentialOutStream() = default;
virtual bool write(const void* data, size_t size) = 0;
};
class SequentialInStream {
public:
virtual ~SequentialInStream() = default;
virtual bool read(void* data, size_t size, size_t* processedSize) = 0;
};
struct CSeqInStreamWrap {
ISeqInStream vt;
std::unique_ptr<SequentialInStream> inStream;
};
struct CSeqOutStreamWrap {
ISeqOutStream vt;
std::unique_ptr<SequentialOutStream> outStream;
};
class BuffPtrInStream : public SequentialInStream {
public:
explicit BuffPtrInStream(const uint8_t* buffer, size_t bufferSize)
: buffer(buffer), bufferSize(bufferSize) {
}
bool read(void* data, size_t size, size_t* processedSize) override {
if (processedSize) {
*processedSize = 0;
}
if (size == 0 || position >= bufferSize) {
return true;
}
auto remain = bufferSize - position;
if (remain > size) {
remain = size;
}
memcpy(data, static_cast<const uint8_t*>(buffer) + position, remain);
position += remain;
if (processedSize) {
*processedSize = remain;
}
return true;
}
private:
const uint8_t* buffer = nullptr;
size_t bufferSize = 0;
size_t position = 0;
};
class VectorOutStream : public SequentialOutStream {
public:
explicit VectorOutStream(std::vector<uint8_t>* buffer) : buffer(buffer) {
}
bool write(const void* data, size_t size) override {
auto oldSize = buffer->size();
buffer->resize(oldSize + size);
memcpy(&(*buffer)[oldSize], data, size);
return true;
}
private:
std::vector<uint8_t>* buffer;
};
class BuffPtrSeqOutStream : public SequentialOutStream {
public:
BuffPtrSeqOutStream(uint8_t* buffer, size_t size) : buffer(buffer), bufferSize(size) {
}
bool write(const void* data, size_t size) override {
auto remain = bufferSize - position;
if (remain > size) {
remain = size;
}
if (remain != 0) {
memcpy(buffer + position, data, remain);
position += remain;
}
return remain != 0 || size == 0;
}
private:
uint8_t* buffer = nullptr;
size_t bufferSize = 0;
size_t position = 0;
};
static const size_t kStreamStepSize = 1 << 31;
static SRes MyRead(const ISeqInStream* p, void* data, size_t* size) {
CSeqInStreamWrap* wrap = CONTAINER_FROM_VTBL(p, CSeqInStreamWrap, vt);
auto curSize = (*size < kStreamStepSize) ? *size : kStreamStepSize;
if (!wrap->inStream->read(data, curSize, &curSize)) {
return SZ_ERROR_READ;
}
*size = curSize;
return SZ_OK;
}
static size_t MyWrite(const ISeqOutStream* p, const void* buf, size_t size) {
auto* wrap = CONTAINER_FROM_VTBL(p, CSeqOutStreamWrap, vt);
if (wrap->outStream->write(buf, size)) {
return size;
}
return 0;
}
class Lzma2Encoder {
public:
Lzma2Encoder() {
encoder = Lzma2Enc_Create(&gAllocFuncs, &gAllocFuncs);
}
~Lzma2Encoder() {
Lzma2Enc_Destroy(encoder);
}
std::shared_ptr<Data> code(const std::shared_ptr<Data>& inputData) {
if (encoder == nullptr || inputData == nullptr || inputData->size() == 0) {
return nullptr;
}
auto inputSize = inputData->size();
CLzma2EncProps lzma2Props;
Lzma2EncProps_Init(&lzma2Props);
lzma2Props.lzmaProps.dictSize = inputSize;
lzma2Props.lzmaProps.level = 9;
lzma2Props.numTotalThreads = 4;
Lzma2Enc_SetProps(encoder, &lzma2Props);
std::vector<uint8_t> outBuf;
outBuf.resize(1 + 8);
outBuf[0] = Lzma2Enc_WriteProperties(encoder);
for (int i = 0; i < 8; i++) {
outBuf[1 + i] = static_cast<uint8_t>(inputSize >> (8 * i));
}
CSeqInStreamWrap inWrap = {};
inWrap.vt.Read = MyRead;
inWrap.inStream = std::make_unique<BuffPtrInStream>(
static_cast<const uint8_t*>(inputData->data()), inputSize);
CSeqOutStreamWrap outStream = {};
outStream.vt.Write = MyWrite;
outStream.outStream = std::make_unique<VectorOutStream>(&outBuf);
auto status =
Lzma2Enc_Encode2(encoder, &outStream.vt, nullptr, nullptr, &inWrap.vt, nullptr, 0, nullptr);
if (status != SZ_OK) {
return nullptr;
}
return Data::MakeWithCopy(&outBuf[0], outBuf.size());
}
private:
CLzma2EncHandle encoder = nullptr;
};
std::shared_ptr<Data> LzmaUtil::Compress(const std::shared_ptr<Data>& pixelData) {
Lzma2Encoder encoder;
return encoder.code(pixelData);
}
class Lzma2Decoder {
public:
Lzma2Decoder() {
decoder = Lzma2DecMt_Create(&gAllocFuncs, &gAllocFuncs);
}
~Lzma2Decoder() {
if (decoder) {
Lzma2DecMt_Destroy(decoder);
}
}
std::shared_ptr<Data> code(const std::shared_ptr<Data>& inputData) {
if (decoder == nullptr || inputData == nullptr || inputData->size() == 0) {
return nullptr;
}
auto input = static_cast<const uint8_t*>(inputData->data());
auto inputSize = inputData->size() - 9;
Byte prop = static_cast<const Byte*>(input)[0];
CLzma2DecMtProps props;
Lzma2DecMtProps_Init(&props);
props.inBufSize_ST = inputSize;
props.numThreads = 1;
UInt64 outBufferSize = 0;
for (int i = 0; i < 8; i++) {
outBufferSize |= (input[1 + i] << (i * 8));
}
auto outBuffer = new uint8_t[outBufferSize];
CSeqInStreamWrap inWrap = {};
inWrap.vt.Read = MyRead;
inWrap.inStream = std::make_unique<BuffPtrInStream>(input + 9, inputSize);
CSeqOutStreamWrap outWrap = {};
outWrap.vt.Write = MyWrite;
outWrap.outStream = std::make_unique<BuffPtrSeqOutStream>(outBuffer, outBufferSize);
UInt64 inProcessed = 0;
int isMT = false;
auto res = Lzma2DecMt_Decode(decoder, prop, &props, &outWrap.vt, &outBufferSize, 1, &inWrap.vt,
&inProcessed, &isMT, nullptr);
if (res == SZ_OK && inputSize == inProcessed) {
return Data::MakeAdopted(outBuffer, outBufferSize, Data::DeleteProc);
}
delete[] outBuffer;
return nullptr;
}
private:
CLzma2DecMtHandle decoder = nullptr;
};
std::shared_ptr<Data> LzmaUtil::Decompress(const std::shared_ptr<Data>& data) {
Lzma2Decoder decoder;
return decoder.code(data);
}
} // namespace pag

Simple sound wave generator with SDL in c++

i am having problems understanding how the audio part of the sdl library works
now, i know that when you initialize it, you have to specify the frequency and a >>callback<< function, which i think is then called automatically at the given frequency.
can anyone who worked with the sdl library write a simple example that would use sdl_audio to generate a 440 hz square wave (since it is the simplest waveform) at a sampling frequency of 44000 hz?
The Introduction to SDL (2011 cached version: 2) has got a neat example of using SDL Sound library that should get you started: http://www.libsdl.org/intro.en/usingsound.html
EDIT: Here is a working program that does what you asked for. I modified a bit the code found here: http://www.dgames.org/beep-sound-with-sdl/
#include <SDL/SDL.h>
#include <SDL/SDL_audio.h>
#include <queue>
#include <cmath>
const int AMPLITUDE = 28000;
const int FREQUENCY = 44100;
struct BeepObject
{
double freq;
int samplesLeft;
};
class Beeper
{
private:
double v;
std::queue<BeepObject> beeps;
public:
Beeper();
~Beeper();
void beep(double freq, int duration);
void generateSamples(Sint16 *stream, int length);
void wait();
};
void audio_callback(void*, Uint8*, int);
Beeper::Beeper()
{
SDL_AudioSpec desiredSpec;
desiredSpec.freq = FREQUENCY;
desiredSpec.format = AUDIO_S16SYS;
desiredSpec.channels = 1;
desiredSpec.samples = 2048;
desiredSpec.callback = audio_callback;
desiredSpec.userdata = this;
SDL_AudioSpec obtainedSpec;
// you might want to look for errors here
SDL_OpenAudio(&desiredSpec, &obtainedSpec);
// start play audio
SDL_PauseAudio(0);
}
Beeper::~Beeper()
{
SDL_CloseAudio();
}
void Beeper::generateSamples(Sint16 *stream, int length)
{
int i = 0;
while (i < length) {
if (beeps.empty()) {
while (i < length) {
stream[i] = 0;
i++;
}
return;
}
BeepObject& bo = beeps.front();
int samplesToDo = std::min(i + bo.samplesLeft, length);
bo.samplesLeft -= samplesToDo - i;
while (i < samplesToDo) {
stream[i] = AMPLITUDE * std::sin(v * 2 * M_PI / FREQUENCY);
i++;
v += bo.freq;
}
if (bo.samplesLeft == 0) {
beeps.pop();
}
}
}
void Beeper::beep(double freq, int duration)
{
BeepObject bo;
bo.freq = freq;
bo.samplesLeft = duration * FREQUENCY / 1000;
SDL_LockAudio();
beeps.push(bo);
SDL_UnlockAudio();
}
void Beeper::wait()
{
int size;
do {
SDL_Delay(20);
SDL_LockAudio();
size = beeps.size();
SDL_UnlockAudio();
} while (size > 0);
}
void audio_callback(void *_beeper, Uint8 *_stream, int _length)
{
Sint16 *stream = (Sint16*) _stream;
int length = _length / 2;
Beeper* beeper = (Beeper*) _beeper;
beeper->generateSamples(stream, length);
}
int main(int argc, char* argv[])
{
SDL_Init(SDL_INIT_AUDIO);
int duration = 1000;
double Hz = 440;
Beeper b;
b.beep(Hz, duration);
b.wait();
return 0;
}
Good luck.
A boiled-down variant of the beeper-example, reduced to the bare minimum (with error-handling).
#include <math.h>
#include <SDL.h>
#include <SDL_audio.h>
const int AMPLITUDE = 28000;
const int SAMPLE_RATE = 44100;
void audio_callback(void *user_data, Uint8 *raw_buffer, int bytes)
{
Sint16 *buffer = (Sint16*)raw_buffer;
int length = bytes / 2; // 2 bytes per sample for AUDIO_S16SYS
int &sample_nr(*(int*)user_data);
for(int i = 0; i < length; i++, sample_nr++)
{
double time = (double)sample_nr / (double)SAMPLE_RATE;
buffer[i] = (Sint16)(AMPLITUDE * sin(2.0f * M_PI * 441.0f * time)); // render 441 HZ sine wave
}
}
int main(int argc, char *argv[])
{
if(SDL_Init(SDL_INIT_AUDIO) != 0) SDL_Log("Failed to initialize SDL: %s", SDL_GetError());
int sample_nr = 0;
SDL_AudioSpec want;
want.freq = SAMPLE_RATE; // number of samples per second
want.format = AUDIO_S16SYS; // sample type (here: signed short i.e. 16 bit)
want.channels = 1; // only one channel
want.samples = 2048; // buffer-size
want.callback = audio_callback; // function SDL calls periodically to refill the buffer
want.userdata = &sample_nr; // counter, keeping track of current sample number
SDL_AudioSpec have;
if(SDL_OpenAudio(&want, &have) != 0) SDL_LogError(SDL_LOG_CATEGORY_AUDIO, "Failed to open audio: %s", SDL_GetError());
if(want.format != have.format) SDL_LogError(SDL_LOG_CATEGORY_AUDIO, "Failed to get the desired AudioSpec");
SDL_PauseAudio(0); // start playing sound
SDL_Delay(1000); // wait while sound is playing
SDL_PauseAudio(1); // stop playing sound
SDL_CloseAudio();
return 0;
}
SDL 2 C example
The following code produces a sinusoidal sound, it is adapted from: https://codereview.stackexchange.com/questions/41086/play-some-sine-waves-with-sdl2
main.c
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <SDL2/SDL.h>
const double ChromaticRatio = 1.059463094359295264562;
const double Tao = 6.283185307179586476925;
Uint32 sampleRate = 48000;
Uint32 frameRate = 60;
Uint32 floatStreamLength = 1024;
Uint32 samplesPerFrame;
Uint32 msPerFrame;
double practicallySilent = 0.001;
Uint32 audioBufferLength = 48000;
float *audioBuffer;
SDL_atomic_t audioCallbackLeftOff;
Sint32 audioMainLeftOff;
Uint8 audioMainAccumulator;
SDL_AudioDeviceID AudioDevice;
SDL_AudioSpec audioSpec;
SDL_Event event;
SDL_bool running = SDL_TRUE;
typedef struct {
float *waveform;
Uint32 waveformLength;
double volume;
double pan;
double frequency;
double phase;
} voice;
void speak(voice *v) {
float sample;
Uint32 sourceIndex;
double phaseIncrement = v->frequency/sampleRate;
Uint32 i;
if (v->volume > practicallySilent) {
for (i = 0; (i + 1) < samplesPerFrame; i += 2) {
v->phase += phaseIncrement;
if (v->phase > 1)
v->phase -= 1;
sourceIndex = v->phase*v->waveformLength;
sample = v->waveform[sourceIndex]*v->volume;
audioBuffer[audioMainLeftOff+i] += sample*(1-v->pan);
audioBuffer[audioMainLeftOff+i+1] += sample*v->pan;
}
}
else {
for (i=0; i<samplesPerFrame; i+=1)
audioBuffer[audioMainLeftOff+i] = 0;
}
audioMainAccumulator++;
}
double getFrequency(double pitch) {
return pow(ChromaticRatio, pitch-57)*440;
}
int getWaveformLength(double pitch) {
return sampleRate / getFrequency(pitch)+0.5f;
}
void buildSineWave(float *data, Uint32 length) {
Uint32 i;
for (i=0; i < length; i++)
data[i] = sin(i*(Tao/length));
}
void logSpec(SDL_AudioSpec *as) {
printf(
" freq______%5d\n"
" format____%5d\n"
" channels__%5d\n"
" silence___%5d\n"
" samples___%5d\n"
" size______%5d\n\n",
(int) as->freq,
(int) as->format,
(int) as->channels,
(int) as->silence,
(int) as->samples,
(int) as->size
);
}
void logVoice(voice *v) {
printf(
" waveformLength__%d\n"
" volume__________%f\n"
" pan_____________%f\n"
" frequency_______%f\n"
" phase___________%f\n",
v->waveformLength,
v->volume,
v->pan,
v->frequency,
v->phase
);
}
void logWavedata(float *floatStream, Uint32 floatStreamLength, Uint32 increment) {
printf("\n\nwaveform data:\n\n");
Uint32 i=0;
for (i = 0; i < floatStreamLength; i += increment)
printf("%4d:%2.16f\n", i, floatStream[i]);
printf("\n\n");
}
void audioCallback(void *unused, Uint8 *byteStream, int byteStreamLength) {
float* floatStream = (float*) byteStream;
Sint32 localAudioCallbackLeftOff = SDL_AtomicGet(&audioCallbackLeftOff);
Uint32 i;
for (i = 0; i < floatStreamLength; i++) {
floatStream[i] = audioBuffer[localAudioCallbackLeftOff];
localAudioCallbackLeftOff++;
if (localAudioCallbackLeftOff == audioBufferLength)
localAudioCallbackLeftOff = 0;
}
SDL_AtomicSet(&audioCallbackLeftOff, localAudioCallbackLeftOff);
}
int init(void) {
SDL_Init(SDL_INIT_AUDIO | SDL_INIT_TIMER);
SDL_AudioSpec want;
SDL_zero(want);
want.freq = sampleRate;
want.format = AUDIO_F32;
want.channels = 2;
want.samples = floatStreamLength;
want.callback = audioCallback;
AudioDevice = SDL_OpenAudioDevice(NULL, 0, &want, &audioSpec, SDL_AUDIO_ALLOW_FORMAT_CHANGE);
if (AudioDevice == 0) {
printf("\nFailed to open audio: %s\n", SDL_GetError());
return 1;
}
printf("want:\n");
logSpec(&want);
printf("audioSpec:\n");
logSpec(&audioSpec);
if (audioSpec.format != want.format) {
printf("\nCouldn't get Float32 audio format.\n");
return 2;
}
sampleRate = audioSpec.freq;
floatStreamLength = audioSpec.size / 4;
samplesPerFrame = sampleRate / frameRate;
msPerFrame = 1000 / frameRate;
audioMainLeftOff = samplesPerFrame * 8;
SDL_AtomicSet(&audioCallbackLeftOff, 0);
if (audioBufferLength % samplesPerFrame)
audioBufferLength += samplesPerFrame - (audioBufferLength % samplesPerFrame);
audioBuffer = malloc(sizeof(float) * audioBufferLength);
return 0;
}
int onExit(void) {
SDL_CloseAudioDevice(AudioDevice);
SDL_Quit();
return 0;
}
int main(int argc, char *argv[]) {
float syncCompensationFactor = 0.0016;
Sint32 mainAudioLead;
Uint32 i;
voice testVoiceA;
voice testVoiceB;
voice testVoiceC;
testVoiceA.volume = 1;
testVoiceB.volume = 1;
testVoiceC.volume = 1;
testVoiceA.pan = 0.5;
testVoiceB.pan = 0;
testVoiceC.pan = 1;
testVoiceA.phase = 0;
testVoiceB.phase = 0;
testVoiceC.phase = 0;
testVoiceA.frequency = getFrequency(45);
testVoiceB.frequency = getFrequency(49);
testVoiceC.frequency = getFrequency(52);
Uint16 C0waveformLength = getWaveformLength(0);
testVoiceA.waveformLength = C0waveformLength;
testVoiceB.waveformLength = C0waveformLength;
testVoiceC.waveformLength = C0waveformLength;
float sineWave[C0waveformLength];
buildSineWave(sineWave, C0waveformLength);
testVoiceA.waveform = sineWave;
testVoiceB.waveform = sineWave;
testVoiceC.waveform = sineWave;
if (init())
return 1;
SDL_Delay(42);
SDL_PauseAudioDevice(AudioDevice, 0);
while (running) {
while (SDL_PollEvent(&event) != 0) {
if (event.type == SDL_QUIT) {
running = SDL_FALSE;
}
}
for (i = 0; i < samplesPerFrame; i++)
audioBuffer[audioMainLeftOff+i] = 0;
speak(&testVoiceA);
speak(&testVoiceB);
speak(&testVoiceC);
if (audioMainAccumulator > 1) {
for (i=0; i<samplesPerFrame; i++) {
audioBuffer[audioMainLeftOff+i] /= audioMainAccumulator;
}
}
audioMainAccumulator = 0;
audioMainLeftOff += samplesPerFrame;
if (audioMainLeftOff == audioBufferLength)
audioMainLeftOff = 0;
mainAudioLead = audioMainLeftOff - SDL_AtomicGet(&audioCallbackLeftOff);
if (mainAudioLead < 0)
mainAudioLead += audioBufferLength;
if (mainAudioLead < floatStreamLength)
printf("An audio collision may have occured!\n");
SDL_Delay(mainAudioLead * syncCompensationFactor);
}
onExit();
return 0;
}
Compile and run:
gcc -ggdb3 -O3 -std=c99 -Wall -Wextra -pedantic -o main.out main.c -lSDL2 -lm
./main.out
Should be easy to turn this into a simple piano with: https://github.com/cirosantilli/cpp-cheat/blob/f734a2e76fbcfc67f707ae06be7a2a2ef5db47d1/c/interactive/audio_gen.c#L44
For wav manipulation, also check the official examples:
http://hg.libsdl.org/SDL/file/e12c38730512/test/testresample.c
http://hg.libsdl.org/SDL/file/e12c38730512/test/loopwave.c
Tested on Ubuntu 19.10, SDL 2.0.10.
This is a minimal example of how to play a sine wave in SDL2.
Make sure to call SDL_Init(SDL_INIT_AUDIO) before creating an instance of Sound.
Sound.h
#include <cstdint>
#include <SDL2/SDL.h>
class Sound
{
public:
Sound();
~Sound();
void play();
void stop();
const double m_sineFreq;
const double m_sampleFreq;
const double m_samplesPerSine;
uint32_t m_samplePos;
private:
static void SDLAudioCallback(void *data, Uint8 *buffer, int length);
SDL_AudioDeviceID m_device;
};
Sound.cpp
#include "Sound.h"
#include <cmath>
#include <iostream>
Sound::Sound()
: m_sineFreq(1000),
m_sampleFreq(44100),
m_samplesPerSine(m_sampleFreq / m_sineFreq),
m_samplePos(0)
{
SDL_AudioSpec wantSpec, haveSpec;
SDL_zero(wantSpec);
wantSpec.freq = m_sampleFreq;
wantSpec.format = AUDIO_U8;
wantSpec.channels = 1;
wantSpec.samples = 2048;
wantSpec.callback = SDLAudioCallback;
wantSpec.userdata = this;
m_device = SDL_OpenAudioDevice(NULL, 0, &wantSpec, &haveSpec, SDL_AUDIO_ALLOW_FORMAT_CHANGE);
if (m_device == 0)
{
std::cout << "Failed to open audio: " << SDL_GetError() << std::endl;
}
}
Sound::~Sound()
{
SDL_CloseAudioDevice(m_device);
}
void Sound::play()
{
SDL_PauseAudioDevice(m_device, 0);
}
void Sound::stop()
{
SDL_PauseAudioDevice(m_device, 1);
}
void Sound::SDLAudioCallback(void *data, Uint8 *buffer, int length)
{
Sound *sound = reinterpret_cast<Sound*>(data);
for(int i = 0; i < length; ++i)
{
buffer[i] = (std::sin(sound->m_samplePos / sound->m_samplesPerSine * M_PI * 2) + 1) * 127.5;
++sound->m_samplePos;
}
}