I'm trying to write a program that accepts a webM file (media) as a parameter, and then output the stream details via TTY in as much detail as possible. I figured I'd try to open the file in binary mode, but am not sure where to start.
Thanks for the help.
This sounds like a very high level question, so I'll answer it as such:
If you're creating a command line program in C/C++ that needs to accept parameters, look up how to use the 'argc' and 'argv' parameters to the main() function.
Once you have the parameter being passed into your main function, you will try to open it using some file reading library (there are several to choose from based on your needs and platform). Yes, you will want to open a WebM file in binary mode if the file library cares about the difference. If using fopen(), specify "rb" to read in binary mode-- this won't make any difference on Unix (vs. plain "r") but it will make a big difference on Windows.
From there, you can start reading bytes from the WebM file and processing them. Be advised that WebM is based on the Matroska multimedia format which is quite involved. If you are doing this as an academic exercise, more power to you. If you are looking to get something accomplished on a tight deadline, there are libraries you can call to do the heavy lifting of Matroska parsing on your behalf.
You can do this by making use of libwebm. The sample code is given below. It prints header, cluster, segments etc.
main.cpp
#include "stdio.h"
#include "stdlib.h"
#include "stdbool.h"
#include "string.h"
#include <memory>
#include <mkv/mkvreader.hpp>
#include <mkv/mkvparser.hpp>
#include <mkv/mkvparser.hpp>
#include "webm_parser.h"
static const wchar_t* utf8towcs(const char* str);
bool InputHasCues(const mkvparser::Segment* const segment);
using namespace mkvparser;
/**
* This file reads an webm file. Generates a new file with random number
* of packets in a single webm page.
*/
int webm_parse(int argc, char **argv)
{
int ret = -1;
char *file_out;
char *file_in;
FILE *fd_out = NULL;
MkvReader reader;
if(argc != 3)
{
printf("Usage: ./webm <input webm file> <output webm file>\n");
exit(0);
}
file_in = argv[1];
file_out = argv[2];
printf("\n\nInput webm file = %s , Output webm file = %s\n", file_in, file_out);
fd_out = fopen(file_out, "w+");
if(fd_out == NULL) goto on_error;
if(reader.Open(file_in))
{
printf("Error opening input file %s", file_in);
}
else
{
printf("Successfully opened input file %s\n", file_in);
}
webm_parse_header(&reader);
/** Return 0 on success */
printf("\n");
return ret;
on_error:
if(fd_out) fclose(fd_out);
printf("Error while parse/generate webm file\n");
/** Return -1 on failure */
return -1;
}
int webm_parse_header(void *reader)
{
int maj, min, build, rev;
long long pos = 0;
typedef mkvparser::Segment seg_t;
seg_t* pSegment_;
long long ret;
MkvReader *mkvrdr = (MkvReader *)reader;
EBMLHeader ebmlHeader;
GetVersion(maj, min, build, rev);
printf("libmkv verison: %d.%d.%d.%d\n", maj, min, build, rev);
ebmlHeader.Parse(mkvrdr, pos);
printf("\t\t\t EBML Header\n");
printf("\t\tEBML Version\t\t: %lld\n", ebmlHeader.m_version);
printf("\t\tEBML MaxIDLength\t: %lld\n", ebmlHeader.m_maxIdLength);
printf("\t\tEBML MaxSizeLength\t: %lld\n", ebmlHeader.m_maxSizeLength);
printf("\t\tDoc Type\t\t: %s\n", ebmlHeader.m_docType);
printf("\t\tPos\t\t\t: %lld\n", pos);
ret = seg_t::CreateInstance(mkvrdr, pos, pSegment_);
if (ret)
{
printf("Segment::CreateInstance() failed.\n");
return -1;
}
else
{
printf("Segment::CreateInstance() successful.\n");
}
const std::auto_ptr<seg_t> pSegment(pSegment_);
ret = pSegment->Load();
if (ret < 0)
{
printf("Segment::Load() failed.\n");
return -1;
}
else
{
printf("Segment::Load() successful.\n");
}
const SegmentInfo* const pSegmentInfo = pSegment->GetInfo();
const long long timeCodeScale = pSegmentInfo->GetTimeCodeScale();
const long long duration_ns = pSegmentInfo->GetDuration();
const char* const pTitle_ = pSegmentInfo->GetTitleAsUTF8();
const wchar_t* const pTitle = utf8towcs(pTitle_);
const char* const pMuxingApp_ = pSegmentInfo->GetMuxingAppAsUTF8();
const wchar_t* const pMuxingApp = utf8towcs(pMuxingApp_);
const char* const pWritingApp_ = pSegmentInfo->GetWritingAppAsUTF8();
const wchar_t* const pWritingApp = utf8towcs(pWritingApp_);
printf("\n");
printf("\t\t\t Segment Info\n");
printf("\t\tTimeCodeScale\t\t: %lld \n", timeCodeScale);
printf("\t\tDuration\t\t: %lld\n", duration_ns);
const double duration_sec = double(duration_ns) / 1000000000;
printf("\t\tDuration(secs)\t\t: %7.3lf\n", duration_sec);
if (pTitle == NULL)
printf("\t\tTrack Name\t\t: NULL\n");
else
{
printf("\t\tTrack Name\t\t: %ls\n", pTitle);
delete[] pTitle;
}
if (pMuxingApp == NULL)
printf("\t\tMuxing App\t\t: NULL\n");
else
{
printf("\t\tMuxing App\t\t: %ls\n", pMuxingApp);
delete[] pMuxingApp;
}
if (pWritingApp == NULL)
printf("\t\tWriting App\t\t: NULL\n");
else
{
printf("\t\tWriting App\t\t: %ls\n", pWritingApp);
delete[] pWritingApp;
}
// pos of segment payload
printf("\t\tPosition(Segment)\t: %lld\n", pSegment->m_start);
// size of segment payload
printf("\t\tSize(Segment)\t\t: %lld\n", pSegment->m_size);
const mkvparser::Tracks* pTracks = pSegment->GetTracks();
unsigned long track_num = 0;
const unsigned long num_tracks = pTracks->GetTracksCount();
printf("\n\t\t\t Track Info\n");
while (track_num != num_tracks)
{
const Track* const pTrack = pTracks->GetTrackByIndex(track_num++);
if (pTrack == NULL)
continue;
const long trackType = pTrack->GetType();
const long trackNumber = pTrack->GetNumber();
const unsigned long long trackUid = pTrack->GetUid();
const wchar_t* const pTrackName = utf8towcs(pTrack->GetNameAsUTF8());
printf("\t\tTrack Type\t\t: %ld\n", trackType);
printf("\t\tTrack Number\t\t: %ld\n", trackNumber);
printf("\t\tTrack Uid\t\t: %lld\n", trackUid);
if (pTrackName == NULL)
printf("\t\tTrack Name\t\t: NULL\n");
else
{
printf("\t\tTrack Name\t\t: %ls \n", pTrackName);
delete[] pTrackName;
}
const char* const pCodecId = pTrack->GetCodecId();
if (pCodecId == NULL)
printf("\t\tCodec Id\t\t: NULL\n");
else
printf("\t\tCodec Id\t\t: %s\n", pCodecId);
const char* const pCodecName_ = pTrack->GetCodecNameAsUTF8();
const wchar_t* const pCodecName = utf8towcs(pCodecName_);
if (pCodecName == NULL)
printf("\t\tCodec Name\t\t: NULL\n");
else
{
printf("\t\tCodec Name\t\t: %ls\n", pCodecName);
delete[] pCodecName;
}
if (trackType == mkvparser::Track::kVideo)
{
const VideoTrack* const pVideoTrack =
static_cast<const VideoTrack*>(pTrack);
const long long width = pVideoTrack->GetWidth();
printf("\t\tVideo Width\t\t: %lld\n", width);
const long long height = pVideoTrack->GetHeight();
printf("\t\tVideo Height\t\t: %lld\n", height);
const double rate = pVideoTrack->GetFrameRate();
printf("\t\tVideo Rate\t\t: %f\n", rate);
}
if (trackType == mkvparser::Track::kAudio)
{
const AudioTrack* const pAudioTrack =
static_cast<const AudioTrack*>(pTrack);
const long long channels = pAudioTrack->GetChannels();
printf("\t\tAudio Channels\t\t: %lld\n", channels);
const long long bitDepth = pAudioTrack->GetBitDepth();
printf("\t\tAudio BitDepth\t\t: %lld\n", bitDepth);
const double sampleRate = pAudioTrack->GetSamplingRate();
printf("\t\tAddio Sample Rate\t: %.3f\n", sampleRate);
const long long codecDelay = pAudioTrack->GetCodecDelay();
printf("\t\tAudio Codec Delay\t: %lld\n", codecDelay);
const long long seekPreRoll = pAudioTrack->GetSeekPreRoll();
printf("\t\tAudio Seek Pre Roll\t: %lld\n", seekPreRoll);
}
}
printf("\n\n\t\t\t Cluster Info\n");
const unsigned long clusterCount = pSegment->GetCount();
printf("\t\tCluster Count\t: %ld\n\n", clusterCount);
if (clusterCount == 0)
{
printf("\t\tSegment has no clusters.\n");
return -1;
}
const mkvparser::Cluster* pCluster = pSegment->GetFirst();
while ((pCluster != NULL) && !pCluster->EOS())
{
const long long timeCode = pCluster->GetTimeCode();
printf("\t\tCluster Time Code\t: %lld\n", timeCode);
const long long time_ns = pCluster->GetTime();
printf("\t\tCluster Time (ns)\t: %lld\n", time_ns);
const BlockEntry* pBlockEntry;
long status = pCluster->GetFirst(pBlockEntry);
if (status < 0) // error
{
printf("\t\tError parsing first block of cluster\n");
fflush(stdout);
return -1;
}
while ((pBlockEntry != NULL) && !pBlockEntry->EOS())
{
const Block* const pBlock = pBlockEntry->GetBlock();
const long long trackNum = pBlock->GetTrackNumber();
const unsigned long tn = static_cast<unsigned long>(trackNum);
const Track* const pTrack = pTracks->GetTrackByNumber(tn);
if (pTrack == NULL)
printf("\t\t\tBlock\t\t:UNKNOWN TRACK TYPE\n");
else
{
const long long trackType = pTrack->GetType();
const int frameCount = pBlock->GetFrameCount();
const long long time_ns = pBlock->GetTime(pCluster);
const long long discard_padding = pBlock->GetDiscardPadding();
printf("\t\t\tBlock\t\t:%s,%s,%15lld,%lld\n",
(trackType == mkvparser::Track::kVideo) ? "V" : "A",
pBlock->IsKey() ? "I" : "P", time_ns, discard_padding);
for (int i = 0; i < frameCount; ++i)
{
const Block::Frame& theFrame = pBlock->GetFrame(i);
const long size = theFrame.len;
const long long offset = theFrame.pos;
printf("\t\t\t %15ld,%15llx\n", size, offset);
}
}
status = pCluster->GetNext(pBlockEntry, pBlockEntry);
if (status < 0)
{
printf("\t\t\tError parsing next block of cluster\n");
fflush(stdout);
return -1;
}
}
pCluster = pSegment->GetNext(pCluster);
}
if (InputHasCues(pSegment.get()))
{
// Walk them.
const mkvparser::Cues* const cues = pSegment->GetCues();
const mkvparser::CuePoint* cue = cues->GetFirst();
int cue_point_num = 1;
printf("\t\tCues\n");
do
{
for (track_num = 0; track_num < num_tracks; ++track_num)
{
const mkvparser::Track* const track =
pTracks->GetTrackByIndex(track_num);
const mkvparser::CuePoint::TrackPosition* const track_pos =
cue->Find(track);
if (track_pos != NULL)
{
const char track_type =
(track->GetType() == mkvparser::Track::kVideo) ? 'V' : 'A';
printf(
"\t\t\tCue Point %4d Track %3lu(%c) Time %14lld "
"Block %4lld Pos %8llx\n",
cue_point_num, track->GetNumber(), track_type,
cue->GetTime(pSegment.get()), track_pos->m_block,
track_pos->m_pos);
}
}
cue = cues->GetNext(cue);
++cue_point_num;
} while (cue != NULL);
}
const mkvparser::Tags* const tags = pSegment->GetTags();
if (tags && tags->GetTagCount() > 0)
{
printf("\t\tTags\n");
for (int i = 0; i < tags->GetTagCount(); ++i)
{
const mkvparser::Tags::Tag* const tag = tags->GetTag(i);
printf("\t\t\tTag\n");
for (int j = 0; j < tag->GetSimpleTagCount(); j++)
{
const mkvparser::Tags::SimpleTag* const simple_tag =
tag->GetSimpleTag(j);
printf("\t\t\t\tSimple Tag \"%s\" Value \"%s\"\n",
simple_tag->GetTagName(), simple_tag->GetTagString());
}
}
}
fflush(stdout);
return 0;
on_error:
return -1;
}
static const wchar_t* utf8towcs(const char* str)
{
if (str == NULL)
return NULL;
// TODO: this probably requires that the locale be
// configured somehow:
const size_t size = mbstowcs(NULL, str, 0);
if (size == 0)
return NULL;
wchar_t* const val = new wchar_t[size + 1];
mbstowcs(val, str, size);
val[size] = L'\0';
return val;
}
bool InputHasCues(const mkvparser::Segment* const segment)
{
const mkvparser::Cues* const cues = segment->GetCues();
if (cues == NULL)
return false;
while (!cues->DoneParsing())
cues->LoadCuePoint();
const mkvparser::CuePoint* const cue_point = cues->GetFirst();
if (cue_point == NULL)
return false;
return true;
}
Related
I'm in the process of working on a kernel program to handle printing capabilities of input for a custom OS. I'm following Poncho's 2nd YouTube Video series found here, I'm currently on Video 4 in the series where he starts to add numerical types as inputs to the renderer's print function. Now, my code isn't exactly like his as I made some modifications.
-Note- This won't compile directly as there is no main function. _start is being called or invoked by a bootloader that isn't shown here, I will however, add it to the bottom of this question.
When I use the class's print function like this within my kernel:
#include "BasicRenderer.h"
extern "C" void _start(Framebuffer* framebuffer, PSF1_FONT** fonts) {
BasicRenderer = renderer(framebuffer, fonts);
renderer.Print("This is some text");
renderer.Print('\n');
renderer.Print(uint64_t(123456789));
renderer.Print('\n');
renderer.Print(int64_t(-123456789));
return;
}
And I run the kernel in emu. I'm getting the following output displayed:
This is some text
123456789
-123456789
The above is correct, however, when I try to incorporate the ability to parse a newline set of characters being either \n or \0 within of a const char* that acts as a string as seen in the following example:
#include "BasicRenderer.h"
extern "C" void _start(Framebuffer* framebuffer, PSF1_FONT** fonts) {
BasicRenderer = renderer(framebuffer, fonts);
renderer.Print("This is some text\n");
renderer.Print(uint64_t(123456789));
renderer.Print('\n');
renderer.Print(int64_t(-123456789));
return;
}
And now the displayed output is:
This is some text
123456789
-123456789
Here, the output in the second line has a space preceding the numerical value to be displayed after the call to Print() that has a \n within its string. I'm not sure what is causing this in my code. Does it have to do with the while condition or how I'm incrementing and indexing into the character string within BasicRenderer::Print(const char* str)? Or is it coming from BasicRender::PutChar(char c)? Or is it within one of the to_string() functions?
Here is the relevant implementation code...
BasicRenderer.cpp
#include "BasicRenderer.h"
void BasicRenderer::Print(const char* str) {
char* chr = (char*)str;
while(*chr != 0) {
if ( (*chr == '\\') && ((*chr+1 == 'n') || (*chr+1 == '0')) ) {
PutChar('\n');
chr++;
chr++;
} else {
PutChar(*chr);
cursor_position_.x += 8;
if (cursor_position_.x + 8 > framebuffer_->Width) {
cursor_position_.x = 0;
cursor_position_.y += 16;
}
chr++;
}
}
}
void BasicRenderer::Print(uint64_t val) {
const char* str = to_string(val);
Print(str);
}
void BasicRenderer::Print(int64_t val) {
const char* str = to_string(val);
Print(str);
}
void BasicRenderer::PutChar(char c) {
if (c == '\n' || c == '\0') {
cursor_position_.x = 0;
cursor_position_.y += 16;
} else {
unsigned int* pixPtr = (unsigned int*)framebuffer_->BaseAddress;
char* fontPtr = (char*)selected_font_->glyphBuffer + (c * selected_font_->psf1_Header->charsize);
for (unsigned long y = cursor_position_.y; y < cursor_position_.y + 16; y++) {
for (unsigned long x = cursor_position_.x; x < cursor_position_.x + 8; x++) {
if ((*fontPtr & (0b10000000 >> (x - cursor_position_.x))) > 0) {
*(unsigned int*)(pixPtr + x + (y * framebuffer_->PixelsPerScanLine)) = font_color_;
}
}
fontPtr++;
}
}
}
cstr.cpp
#include "cstr.h"
const char* to_string(uint64_t value) {
static char output_uint_buffer[128];
uint8_t size = 0;
uint64_t sizeTest = value;
while (sizeTest / 10 > 0) {
sizeTest /= 10;
size++;
}
uint8_t idx = 0;
while (value / 10 > 0) {
uint8_t remainder = value % 10;
value /= 10;
output_uint_buffer[size - idx] = remainder + '0';
idx++;
}
uint8_t remainder = value % 10;
output_uint_buffer[size-idx] = remainder + '0';
output_uint_buffer[size + 1] = 0;
return output_uint_buffer;
}
const char* to_string(int64_t value) {
static char output_int_buffer[128];
uint8_t isNegative = 0;
if (value < 0) {
isNegative = 1;
value *= -1;
output_int_buffer[0] = '-';
}
uint8_t size = 0;
uint64_t sizeTest = value;
while (sizeTest / 10 > 0) {
sizeTest /= 10;
size++;
}
uint8_t idx = 0;
while (value / 10 > 0) {
uint8_t remainder = value % 10;
value /= 10;
output_int_buffer[isNegative + size - idx] = remainder + '0';
idx++;
}
uint8_t remainder = value % 10;
output_int_buffer[isNegative + size - idx] = remainder + '0';
output_int_buffer[isNegative + size + 1] = 0;
return output_int_buffer;
}
And here is the rest of the declarations...
BasicRender.h
#pragma once
#include "cstr.h"
#include "math.h"
#include "framebuffer.h"
#include "SimpleFonts.h"
class BasicRenderer {
public:
BasicRenderer(Framebuffer* framebuffer, PSF1_FONT** fonts) :
framebuffer_{framebuffer},
fonts_{fonts},
cursor_position_({0,0}),
selected_font_{fonts_[0]},
font_color_{0xFFFFFFFF}
{}
void Print(const char* str);
void Print(char c) { PutChar(c); }
void Print(uint64_t val);
void Print(int64_t val);
private:
void PutChar(char c);
Framebuffer* framebuffer_;
Point cursor_position_;
PSF1_FONT** fonts_;
PSF1_FONT* selected_font_;
unsigned int font_color_;
};
cstr.h
#pragma once
#include <stdint.h>
const char* to_string(uint64_t value);
const char* to_string(int64_t value);
math.h
#pragma once
struct Point {
unsigned int x;
unsigned int y;
};
Framebuffer.h
#pragma once
#include <stddef.h>
struct Framebuffer {
void* BaseAddress;
size_t BufferSize;
unsigned int Width;
unsigned int Height;
unsigned int PixelsPerScanLine;
};
SimpleFonts.h
#pragma once
struct PSF1_HEADER {
unsigned char magic[2];
unsigned char mode;
unsigned char charsize;
};
struct PSF1_FONT {
PSF1_HEADER* psf1_Header;
void* glyphBuffer;
};
Here is the bootloader application that invokes the above kernel.
main.c
#include <efi.h>
#include <efilib.h>
#include <elf.h>
#define PSF1_MAGIC0 0x36
#define PSF1_MAGIC1 0x04
typedef unsigned long long size_t;
typedef struct {
unsigned char magic[2];
unsigned char mode;
unsigned char charsize;
} PSF1_HEADER;
typedef struct {
PSF1_HEADER* psf1_Header;
void* glyphBuffer;
} PSF1_FONT;
typedef struct {
void* BaseAddress;
size_t BufferSize;
unsigned int Width;
unsigned int Height;
unsigned int PixelsPerScanLine;
} Framebuffer; Framebuffer framebuffer;
Framebuffer* InitializeGOP() {
EFI_GUID gopGuid = EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID;
EFI_GRAPHICS_OUTPUT_PROTOCOL* gop;
EFI_STATUS status;
status = uefi_call_wrapper(BS->LocateProtocol, 3, &gopGuid, NULL, (void**)&gop);
if (EFI_ERROR(status)) {
Print(L"Unable to locate GOP\n\r");
return NULL;
} else {
Print(L"GOP located\n\r");
}
framebuffer.BaseAddress = (void*)gop->Mode->FrameBufferBase;
framebuffer.BufferSize = gop->Mode->FrameBufferSize;
framebuffer.Width = gop->Mode->Info->HorizontalResolution;
framebuffer.Height = gop->Mode->Info->VerticalResolution;
framebuffer.PixelsPerScanLine = gop->Mode->Info->PixelsPerScanLine;
return &framebuffer;
}
EFI_FILE* LoadFile(EFI_FILE* Directory, CHAR16* Path, EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable) {
EFI_FILE* LoadedFile;
EFI_LOADED_IMAGE_PROTOCOL* LoadedImage;
SystemTable->BootServices->HandleProtocol(ImageHandle, &gEfiLoadedImageProtocolGuid, (void**)&LoadedImage);
EFI_SIMPLE_FILE_SYSTEM_PROTOCOL* FileSystem;
SystemTable->BootServices->HandleProtocol(LoadedImage->DeviceHandle, &gEfiSimpleFileSystemProtocolGuid, (void**)&FileSystem);
if (Directory == NULL) {
FileSystem->OpenVolume(FileSystem, &Directory);
}
EFI_STATUS s = Directory->Open(Directory, &LoadedFile, Path, EFI_FILE_MODE_READ, EFI_FILE_READ_ONLY);
if (s != EFI_SUCCESS) {
return NULL;
}
return LoadedFile;
}
PSF1_FONT* LoadPSF1Font(EFI_FILE* Directory, CHAR16* Path, EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable) {
EFI_FILE* font = LoadFile(Directory, Path, ImageHandle, SystemTable);
if (font == NULL) return NULL;
PSF1_HEADER* fontHeader;
SystemTable->BootServices->AllocatePool(EfiLoaderData, sizeof(PSF1_HEADER), (void**)&fontHeader);
UINTN size = sizeof(PSF1_HEADER);
font->Read(font, &size, fontHeader);
if (fontHeader->magic[0] != PSF1_MAGIC0 || fontHeader->magic[1] != PSF1_MAGIC1) return NULL;
UINTN glyphBufferSize = fontHeader->charsize * 256;
if (fontHeader->mode == 1) { // 512 glyph mode
glyphBufferSize *= 2;
}
void* glyphBuffer;
font->SetPosition(font, sizeof(PSF1_HEADER));
SystemTable->BootServices->AllocatePool(EfiLoaderData, glyphBufferSize, (void**)&glyphBuffer);
font->Read(font, &glyphBufferSize, glyphBuffer);
PSF1_FONT* finishedFont;
SystemTable->BootServices->AllocatePool(EfiLoaderData, sizeof(PSF1_FONT), (void**)&finishedFont);
finishedFont->psf1_Header = fontHeader;
finishedFont->glyphBuffer = glyphBuffer;
return finishedFont;
}
int memcmp(const void* aptr, const void* bptr, size_t n) {
const unsigned char* a = aptr, *b = bptr;
for (size_t i = 0; i < n; i++) {
if (a[i] < b[i]) return -1;
else if(a[i] > b[i]) return 1;
}
return 0;
}
EFI_STATUS efi_main (EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE *SystemTable) {
InitializeLib(ImageHandle, SystemTable);
Print(L"Hello World!\n\r");
EFI_FILE* Kernel = LoadFile(NULL, L"kernel.elf", ImageHandle, SystemTable);
if ( Kernel == NULL) {
Print(L"Could not load kernel \n\r");
} else {
Print(L"Kernel Loaded Successfully \n\r");
}
Elf64_Ehdr header;
{
UINTN FileInfoSize;
EFI_FILE_INFO* FileInfo;
Kernel->GetInfo(Kernel, &gEfiFileInfoGuid, &FileInfoSize, NULL);
SystemTable->BootServices->AllocatePool(EfiLoaderData, FileInfoSize, (void**)&FileInfo);
Kernel->GetInfo(Kernel, &gEfiFileInfoGuid, &FileInfoSize, (void**)&FileInfo);
UINTN size = sizeof(header);
Kernel->Read(Kernel, &size, &header);
}
if (
memcmp(&header.e_ident[EI_MAG0], ELFMAG, SELFMAG) != 0 ||
header.e_ident[EI_CLASS] != ELFCLASS64 ||
header.e_ident[EI_DATA] != ELFDATA2LSB ||
header.e_type != ET_EXEC ||
header.e_machine != EM_X86_64 ||
header.e_version != EV_CURRENT
) {
Print(L"kernel format is bad\r\n");
} else {
Print(L"kernel header successfully verified\r\n");
}
Elf64_Phdr* phdrs;
{
Kernel->SetPosition(Kernel, header.e_phoff);
UINTN size = header.e_phnum * header.e_phentsize;
SystemTable->BootServices->AllocatePool(EfiLoaderData, size, (void**)&phdrs);
Kernel->Read(Kernel, &size, phdrs);
}
for (
Elf64_Phdr* phdr = phdrs;
(char*)phdr < (char*)phdrs + header.e_phnum * header.e_phentsize;
phdr = (Elf64_Phdr*)((char*)phdr + header.e_phentsize)
) {
switch(phdr->p_type) {
case PT_LOAD: {
int pages = (phdr->p_memsz + 0x1000 - 1) / 0x1000;
Elf64_Addr segment = phdr->p_paddr;
SystemTable->BootServices->AllocatePages(AllocateAddress, EfiLoaderData, pages, &segment);
Kernel->SetPosition(Kernel, phdr->p_offset);
UINTN size = phdr->p_filesz;
Kernel->Read(Kernel, &size, (void*)segment);
break;
}
}
}
Print(L"Kernel Loaded\n\r");
void (*KernelStart)(Framebuffer*, PSF1_FONT**) = ((__attribute__((sysv_abi)) void(*)(Framebuffer*, PSF1_FONT**) ) header.e_entry);
PSF1_FONT* newFont = LoadPSF1Font(NULL, L"zap-light16.psf", ImageHandle, SystemTable);
if (newFont == NULL) {
Print(L"Font is not valid or is not found\n\r");
} else {
Print(L"Font found, char size = %d\n\r", newFont->psf1_Header->charsize);
}
PSF1_FONT* newFontExt = LoadPSF1Font(NULL, L"zap-ext-light16.psf", ImageHandle, SystemTable);
if (newFont == NULL) {
Print(L"Font is not valid or is not found\n\r");
} else {
Print(L"Font found, char size = %d\n\r", newFont->psf1_Header->charsize);
}
PSF1_FONT* fonts[] = {newFont, newFontExt};
Framebuffer* newBuffer = InitializeGOP();
Print(L"Base: 0x%x\n\rSize: 0x%x\n\rWidth: %d\n\rHeight: %d\n\rPixelsPerScanline: %d\n\r",
newBuffer->BaseAddress,
newBuffer->BufferSize,
newBuffer->Width,
newBuffer->Height,
newBuffer->PixelsPerScanLine);
KernelStart(newBuffer, fonts);
return EFI_SUCCESS; // Exit the UEFI application
}
The problem is here:
if ( (*chr == '\\') && ((*chr+1 == 'n') || (*chr+1 == '0')) ) {
PutChar('\n');
chr++;
chr++;
}
...
You should not be parsing out \n since this will be present in the string as a linefeed character. What you want instead is:
if (*chr == '\n') {
PutChar('\n');
chr++;
}
...
Since i am quiet new to C++ and Image processing i have a problem modifying and adding a function to the code.
The requirement is only to switch between the RGB colors.
#include "stdio.h"
#include "stdlib.h"
#include "string.h"
#include "ctype.h"
#include "math.h"
class myImageData
{
private:
int mW;
int mH;
int mCH;
double * mData;
void SkipComments(FILE *fp)
{
int ch;
char line[100];
while ((ch = fgetc(fp)) != EOF && isspace(ch))
;
if (ch == '#')
{
fgets(line, sizeof(line), fp);
SkipComments(fp);
}
else
{
fseek(fp, -1, SEEK_CUR);
}
}
public:
myImageData(void)
{
this->mData = NULL;
}
~myImageData()
{
if (this->mData != NULL)
{
delete[] this->mData;
}
}
void init(int W, int H, int CH)
{
this->mW = W;
this->mH = H;
this->mCH = CH;
if (this->mData != NULL)
delete[] this->mData;
this->mData = new double[(this->mW)*(this->mH)*(this->mCH)];
}
int getWidth(void)
{
return this->mW;
}
int getHeight(void)
{
return this->mH;
}
int getCH(void)
{
return this->mCH;
}
double * getDataPtr(void)
{
return this->mData;
}
double get(int x, int y)
{
return this->mData[y*(this->mW) + x];
}
double get(int x, int y, int CH)
{
return this->mData[this->mCH * (y*(this->mW) + x) + CH];
}
void set(int x, int y, double value)
{
this->mData[y*(this->mW) + x] = value;
}
void set(int x, int y, int CH, double value)
{
this->mData[this->mCH *(y*(this->mW) + x) + CH] = value;
}
void read(const char *filename);
void save(const char *filename);
};
void myImageData::read(const char *filename)
{
FILE *file = fopen(filename, "r");
if (file == NULL){
printf("Cannot open %s\n", filename);
exit(1); //abnormal termination
}
printf("Read an image from: %s\n", filename);
// read ppm/pgm header
char buf[256];
char filetype[256];
int W, H, Range, CH;
fgets(buf, sizeof(buf), file);
sscanf(buf, "%s", filetype);
SkipComments(file);
fgets(buf, sizeof(buf), file);
sscanf(buf, "%d%d", &W, &H);
SkipComments(file);
fgets(buf, sizeof(buf), file);
sscanf(buf, "%d", &Range);
//printf("Header: %s, %d, %d, %d\n", filetype, W, H, Range);
SkipComments(file);
if (strcmp(filetype, "P5") == 0)
{
CH = 1;
}
else if (strcmp(filetype, "P6") == 0)
{
CH = 3;
}
else
{
printf("Unknown image type\n");
exit(1); //abnormal termination
}
if (Range != 255){
printf("Invalid data\n");
exit(1); //abnormal termination
}
// create myImageData class
init(W, H, CH);
// read ppm data
int datalength = this->mW * this->mH * this->mCH;
unsigned char * temp = new unsigned char[datalength];
fread(temp, sizeof(unsigned char), datalength, file);
double * ptr1 = this->mData;
unsigned char *ptr2 = temp;
for (int i = 0; i < datalength; i++){
*ptr1 = (double)*ptr2;
ptr1++;
ptr2++;
}
delete[] temp;
fclose(file);
}
void myImageData::save(const char *filename){
char filenamefull[256];
if (this->mCH == 1){
sprintf(filenamefull, "%s.pgm", filename);
}
else{
sprintf(filenamefull, "%s.ppm", filename);
}
FILE *file = fopen(filenamefull, "w");
printf("Write an image to: %s \n", filenamefull);
if (this->mCH == 1){
fprintf(file, "P5\n");
}
else{
fprintf(file, "P6\n");
}
fprintf(file, "%d %d\n", this->mW, this->mH);
fprintf(file, "255\n");
int datalength = this->mW * this->mH * this->mCH;
unsigned char * temp = new unsigned char[datalength];
double * ptr1 = this->mData;
unsigned char * ptr2 = temp;
for (int i = 0; i < datalength; i++){
double value = *ptr1;
value = round(value);
if (value > 255) value = 255;
if (value < 0) value = 0;
*ptr2 = (unsigned char)value;
ptr1++;
ptr2++;
}
fwrite(temp, sizeof(unsigned char), datalength, file);
delete[] temp;
fprintf(file, "Â¥n");
fclose(file);
}
The errors i am having:
error LNK2019: unresolved external symbol _main referenced in function ___tmainCRTStartup
error LNK1120: 1 unresolved externals
Firstly, you have no main function. No surprise that your code doesn't work.. all you have is a class to load, save and manipulate PPM image files.
You appear to me using Visual Studio, so you'll need a function that looks like this:
int _tmain(int argc, _TCHAR* argv[])
{
myImageData image;
image.read("atestfile.ppm");
// do some stuff to your image
image.write("outputfile.ppm");
}
I'm assuming you have a test image in PPM format you can use here, of course.
Now this is madness:
double * ptr1 = this->mData;
unsigned char * ptr2 = temp;
for (int i = 0; i < datalength; i++){
double value = *ptr1;
value = round(value);
if (value > 255) value = 255;
if (value < 0) value = 0;
*ptr2 = (unsigned char)value;
ptr1++;
ptr2++;
}
You've read from an unsigned char, so there's no point stuffing it into a double, and there's definitely no point in checking if the value lies outside of 0 to 255. Why are you storing things in doubles? It makes no sense! Even if you did do something that needed a full double-precision floating point value per channel, you throw it all away in the output when you clamp everything to 0-255 again:
for (int i = 0; i < datalength; i++){
double value = *ptr1;
value = round(value);
if (value > 255) value = 255;
if (value < 0) value = 0;
*ptr2 = (unsigned char)value;
ptr1++;
ptr2++;
}
Also, this is basically C dressed up in a thin C++ veneer. That's okay, everyone has to start somewhere. But instead of using new to create an array, you can do this:
// read ppm data
int datalength = this->mW * this->mH * this->mCH;
// using a std::vector here means that the allocated memory will be freed
// automatically, even in the result of an error occurring.
std::vector<unsigned char> temp(datalength);
fread(&temp[0], sizeof(unsigned char), datalength, file);
I'd also consider using iostream classes such as fstream instead of fread and fopen and so on. But this is not really the place to get into those kinds of details.
Anyway, What to do with your image once it is loaded? You've got dead easy helper functions to read and write pixel values, which will let you do pretty much anything you want. Here's a simple example, swapping the R and B channels. You might get something better when you actually tell us what you wanted.
void swapRB(myImageData& image)
{
assert(image.getCH() == 3);
for (int x = 0; x < image.getWidth())
{
for (int y = 0; x < image.getHeight())
{
double R = image.get(x, y, 0);
double G = image.get(x, y, 1);
double B = image.get(x, y, 2);
image.set(x, y, 0, B);
image.set(x, y, 2, R);
}
}
}
i have difficulties in using LZMA SDK in my application.
I would like to create a kind of single file compression tool. I dont need any directory support, just need only the LZMA2 stream. But i have no idea on how LZMA SDK is to be used for this.
Please can anyone give me a little example on how the LZMA SDK can be used under C++?
I think that it's a properly little example to use LZMA SDK.
/* LzmaUtil.c -- Test application for LZMA compression
2008-08-05
Igor Pavlov
public domain */
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../LzmaDec.h"
#include "../LzmaEnc.h"
#include "../Alloc.h"
const char *kCantReadMessage = "Can not read input file";
const char *kCantWriteMessage = "Can not write output file";
const char *kCantAllocateMessage = "Can not allocate memory";
const char *kDataErrorMessage = "Data error";
static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
static void SzFree(void *p, void *address) { p = p; MyFree(address); }
static ISzAlloc g_Alloc = { SzAlloc, SzFree };
#define kInBufferSize (1 << 15)
#define kOutBufferSize (1 << 15)
unsigned char g_InBuffer[kInBufferSize];
unsigned char g_OutBuffer[kOutBufferSize];
size_t MyReadFile(FILE *file, void *data, size_t size)
{ return fread(data, 1, size, file); }
int MyReadFileAndCheck(FILE *file, void *data, size_t size)
{ return (MyReadFile(file, data, size) == size); }
size_t MyWriteFile(FILE *file, const void *data, size_t size)
{
if (size == 0)
return 0;
return fwrite(data, 1, size, file);
}
int MyWriteFileAndCheck(FILE *file, const void *data, size_t size)
{ return (MyWriteFile(file, data, size) == size); }
long MyGetFileLength(FILE *file)
{
long length;
fseek(file, 0, SEEK_END);
length = ftell(file);
fseek(file, 0, SEEK_SET);
return length;
}
void PrintHelp(char *buffer)
{
strcat(buffer, "\nLZMA Utility 4.58 Copyright (c) 1999-2008 Igor Pavlov 2008-04-11\n"
"\nUsage: lzma <e|d> inputFile outputFile\n"
" e: encode file\n"
" d: decode file\n");
}
int PrintError(char *buffer, const char *message)
{
strcat(buffer, "\nError: ");
strcat(buffer, message);
strcat(buffer, "\n");
return 1;
}
int PrintErrorNumber(char *buffer, SRes val)
{
sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val);
return 1;
}
int PrintUserError(char *buffer)
{
return PrintError(buffer, "Incorrect command");
}
#define IN_BUF_SIZE (1 << 16)
#define OUT_BUF_SIZE (1 << 16)
static int Decode(FILE *inFile, FILE *outFile, char *rs)
{
UInt64 unpackSize;
int thereIsSize; /* = 1, if there is uncompressed size in headers */
int i;
int res = 0;
CLzmaDec state;
/* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */
unsigned char header[LZMA_PROPS_SIZE + 8];
/* Read and parse header */
if (!MyReadFileAndCheck(inFile, header, sizeof(header)))
return PrintError(rs, kCantReadMessage);
unpackSize = 0;
thereIsSize = 0;
for (i = 0; i < 8; i++)
{
unsigned char b = header[LZMA_PROPS_SIZE + i];
if (b != 0xFF)
thereIsSize = 1;
unpackSize += (UInt64)b << (i * 8);
}
LzmaDec_Construct(&state);
res = LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc);
if (res != SZ_OK)
return res;
{
Byte inBuf[IN_BUF_SIZE];
Byte outBuf[OUT_BUF_SIZE];
size_t inPos = 0, inSize = 0, outPos = 0;
LzmaDec_Init(&state);
for (;;)
{
if (inPos == inSize)
{
inSize = MyReadFile(inFile, inBuf, IN_BUF_SIZE);
inPos = 0;
}
{
SizeT inProcessed = inSize - inPos;
SizeT outProcessed = OUT_BUF_SIZE - outPos;
ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
ELzmaStatus status;
if (thereIsSize && outProcessed > unpackSize)
{
outProcessed = (SizeT)unpackSize;
finishMode = LZMA_FINISH_END;
}
res = LzmaDec_DecodeToBuf(&state, outBuf + outPos, &outProcessed,
inBuf + inPos, &inProcessed, finishMode, &status);
inPos += (UInt32)inProcessed;
outPos += outProcessed;
unpackSize -= outProcessed;
if (outFile != 0)
MyWriteFile(outFile, outBuf, outPos);
outPos = 0;
if (res != SZ_OK || thereIsSize && unpackSize == 0)
break;
if (inProcessed == 0 && outProcessed == 0)
{
if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK)
res = SZ_ERROR_DATA;
break;
}
}
}
}
LzmaDec_Free(&state, &g_Alloc);
return res;
}
typedef struct _CFileSeqInStream
{
ISeqInStream funcTable;
FILE *file;
} CFileSeqInStream;
static SRes MyRead(void *p, void *buf, size_t *size)
{
if (*size == 0)
return SZ_OK;
*size = MyReadFile(((CFileSeqInStream*)p)->file, buf, *size);
/*
if (*size == 0)
return SZE_FAIL;
*/
return SZ_OK;
}
typedef struct _CFileSeqOutStream
{
ISeqOutStream funcTable;
FILE *file;
} CFileSeqOutStream;
static size_t MyWrite(void *pp, const void *buf, size_t size)
{
return MyWriteFile(((CFileSeqOutStream *)pp)->file, buf, size);
}
static SRes Encode(FILE *inFile, FILE *outFile, char *rs)
{
CLzmaEncHandle enc;
SRes res;
CFileSeqInStream inStream;
CFileSeqOutStream outStream;
CLzmaEncProps props;
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
return SZ_ERROR_MEM;
inStream.funcTable.Read = MyRead;
inStream.file = inFile;
outStream.funcTable.Write = MyWrite;
outStream.file = outFile;
LzmaEncProps_Init(&props);
res = LzmaEnc_SetProps(enc, &props);
if (res == SZ_OK)
{
Byte header[LZMA_PROPS_SIZE + 8];
size_t headerSize = LZMA_PROPS_SIZE;
UInt64 fileSize;
int i;
res = LzmaEnc_WriteProperties(enc, header, &headerSize);
fileSize = MyGetFileLength(inFile);
for (i = 0; i < 8; i++)
header[headerSize++] = (Byte)(fileSize >> (8 * i));
if (!MyWriteFileAndCheck(outFile, header, headerSize))
return PrintError(rs, "writing error");
if (res == SZ_OK)
res = LzmaEnc_Encode(enc, &outStream.funcTable, &inStream.funcTable,
NULL, &g_Alloc, &g_Alloc);
}
LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
return res;
}
int main2(int numArgs, const char *args[], char *rs)
{
FILE *inFile = 0;
FILE *outFile = 0;
char c;
int res;
int encodeMode;
if (numArgs == 1)
{
PrintHelp(rs);
return 0;
}
if (numArgs < 3 || numArgs > 4 || strlen(args[1]) != 1)
return PrintUserError(rs);
c = args[1][0];
encodeMode = (c == 'e' || c == 'E');
if (!encodeMode && c != 'd' && c != 'D')
return PrintUserError(rs);
{
size_t t4 = sizeof(UInt32);
size_t t8 = sizeof(UInt64);
if (t4 != 4 || t8 != 8)
return PrintError(rs, "LZMA UTil needs correct UInt32 and UInt64");
}
inFile = fopen(args[2], "rb");
if (inFile == 0)
return PrintError(rs, "Can not open input file");
if (numArgs > 3)
{
outFile = fopen(args[3], "wb+");
if (outFile == 0)
return PrintError(rs, "Can not open output file");
}
else if (encodeMode)
PrintUserError(rs);
if (encodeMode)
{
res = Encode(inFile, outFile, rs);
}
else
{
res = Decode(inFile, outFile, rs);
}
if (outFile != 0)
fclose(outFile);
fclose(inFile);
if (res != SZ_OK)
{
if (res == SZ_ERROR_MEM)
return PrintError(rs, kCantAllocateMessage);
else if (res == SZ_ERROR_DATA)
return PrintError(rs, kDataErrorMessage);
else
return PrintErrorNumber(rs, res);
}
return 0;
}
int MY_CDECL main(int numArgs, const char *args[])
{
char rs[800] = { 0 };
int res = main2(numArgs, args, rs);
printf(rs);
return res;
}
Also you can see it at:
http://read.pudn.com/downloads151/sourcecode/zip/656407/7z460/C/LzmaUtil/LzmaUtil.c__.htm
http://read.pudn.com/downloads157/sourcecode/zip/698262/LZMA/LzmaUtil.c__.htm
I recently found a nice example, written in C++. Credit goes to GH user Treeki who published the original gist:
// note: -D_7ZIP_ST is required when compiling on non-Windows platforms
// g++ -o lzma_sample -std=c++14 -D_7ZIP_ST lzma_sample.cpp LzmaDec.c LzmaEnc.c LzFind.c
#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <memory>
#include "LzmaEnc.h"
#include "LzmaDec.h"
static void *_lzmaAlloc(ISzAllocPtr, size_t size) {
return new uint8_t[size];
}
static void _lzmaFree(ISzAllocPtr, void *addr) {
if (!addr)
return;
delete[] reinterpret_cast<uint8_t *>(addr);
}
static ISzAlloc _allocFuncs = {
_lzmaAlloc, _lzmaFree
};
std::unique_ptr<uint8_t[]> lzmaCompress(const uint8_t *input, uint32_t inputSize, uint32_t *outputSize) {
std::unique_ptr<uint8_t[]> result;
// set up properties
CLzmaEncProps props;
LzmaEncProps_Init(&props);
if (inputSize >= (1 << 20))
props.dictSize = 1 << 20; // 1mb dictionary
else
props.dictSize = inputSize; // smaller dictionary = faster!
props.fb = 40;
// prepare space for the encoded properties
SizeT propsSize = 5;
uint8_t propsEncoded[5];
// allocate some space for the compression output
// this is way more than necessary in most cases...
// but better safe than sorry
// (a smarter implementation would use a growing buffer,
// but this requires a bunch of fuckery that is out of
/// scope for this simple example)
SizeT outputSize64 = inputSize * 1.5;
if (outputSize64 < 1024)
outputSize64 = 1024;
auto output = std::make_unique<uint8_t[]>(outputSize64);
int lzmaStatus = LzmaEncode(
output.get(), &outputSize64, input, inputSize,
&props, propsEncoded, &propsSize, 0,
NULL,
&_allocFuncs, &_allocFuncs);
*outputSize = outputSize64 + 13;
if (lzmaStatus == SZ_OK) {
// tricky: we have to generate the LZMA header
// 5 bytes properties + 8 byte uncompressed size
result = std::make_unique<uint8_t[]>(outputSize64 + 13);
uint8_t *resultData = result.get();
memcpy(resultData, propsEncoded, 5);
for (int i = 0; i < 8; i++)
resultData[5 + i] = (inputSize >> (i * 8)) & 0xFF;
memcpy(resultData + 13, output.get(), outputSize64);
}
return result;
}
std::unique_ptr<uint8_t[]> lzmaDecompress(const uint8_t *input, uint32_t inputSize, uint32_t *outputSize) {
if (inputSize < 13)
return NULL; // invalid header!
// extract the size from the header
UInt64 size = 0;
for (int i = 0; i < 8; i++)
size |= (input[5 + i] << (i * 8));
if (size <= (256 * 1024 * 1024)) {
auto blob = std::make_unique<uint8_t[]>(size);
ELzmaStatus lzmaStatus;
SizeT procOutSize = size, procInSize = inputSize - 13;
int status = LzmaDecode(blob.get(), &procOutSize, &input[13], &procInSize, input, 5, LZMA_FINISH_END, &lzmaStatus, &_allocFuncs);
if (status == SZ_OK && procOutSize == size) {
*outputSize = size;
return blob;
}
}
return NULL;
}
void hexdump(const uint8_t *buf, int size) {
int lines = (size + 15) / 16;
for (int i = 0; i < lines; i++) {
printf("%08x | ", i * 16);
int lineMin = i * 16;
int lineMax = lineMin + 16;
int lineCappedMax = (lineMax > size) ? size : lineMax;
for (int j = lineMin; j < lineCappedMax; j++)
printf("%02x ", buf[j]);
for (int j = lineCappedMax; j < lineMax; j++)
printf(" ");
printf("| ");
for (int j = lineMin; j < lineCappedMax; j++) {
if (buf[j] >= 32 && buf[j] <= 127)
printf("%c", buf[j]);
else
printf(".");
}
printf("\n");
}
}
void testIt(const uint8_t *input, int size) {
printf("Test Input:\n");
hexdump(input, size);
uint32_t compressedSize;
auto compressedBlob = lzmaCompress(input, size, &compressedSize);
if (compressedBlob) {
printf("Compressed:\n");
hexdump(compressedBlob.get(), compressedSize);
} else {
printf("Nope, we screwed it\n");
return;
}
// let's try decompressing it now
uint32_t decompressedSize;
auto decompressedBlob = lzmaDecompress(compressedBlob.get(), compressedSize, &decompressedSize);
if (decompressedBlob) {
printf("Decompressed:\n");
hexdump(decompressedBlob.get(), decompressedSize);
} else {
printf("Nope, we screwed it (part 2)\n");
return;
}
printf("----------\n");
}
void testIt(const char *string) {
testIt((const uint8_t *)string, strlen(string));
}
int main(int argc, char **argv) {
testIt("a");
testIt("here is a cool string");
testIt("here's something that should compress pretty well: abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdef");
return 0;
}
You can refer to this file on how to use lzma2。
https://github.com/Tencent/libpag/blob/aab6391e455193c8ec5b8e2031b495b3fe77b034/test/framework/utils/LzmaUtil.cpp
/////////////////////////////////////////////////////////////////////////////////////////////////
//
// Tencent is pleased to support the open source community by making libpag available.
//
// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// unless required by applicable law or agreed to in writing, software distributed under the
// license is distributed on an "as is" basis, without warranties or conditions of any kind,
// either express or implied. see the license for the specific language governing permissions
// and limitations under the license.
//
/////////////////////////////////////////////////////////////////////////////////////////////////
#include "LzmaUtil.h"
#include "test/framework/lzma/Lzma2DecMt.h"
#include "test/framework/lzma/Lzma2Enc.h"
namespace pag {
static void* LzmaAlloc(ISzAllocPtr, size_t size) {
return new uint8_t[size];
}
static void LzmaFree(ISzAllocPtr, void* address) {
if (!address) {
return;
}
delete[] reinterpret_cast<uint8_t*>(address);
}
static ISzAlloc gAllocFuncs = {LzmaAlloc, LzmaFree};
class SequentialOutStream {
public:
virtual ~SequentialOutStream() = default;
virtual bool write(const void* data, size_t size) = 0;
};
class SequentialInStream {
public:
virtual ~SequentialInStream() = default;
virtual bool read(void* data, size_t size, size_t* processedSize) = 0;
};
struct CSeqInStreamWrap {
ISeqInStream vt;
std::unique_ptr<SequentialInStream> inStream;
};
struct CSeqOutStreamWrap {
ISeqOutStream vt;
std::unique_ptr<SequentialOutStream> outStream;
};
class BuffPtrInStream : public SequentialInStream {
public:
explicit BuffPtrInStream(const uint8_t* buffer, size_t bufferSize)
: buffer(buffer), bufferSize(bufferSize) {
}
bool read(void* data, size_t size, size_t* processedSize) override {
if (processedSize) {
*processedSize = 0;
}
if (size == 0 || position >= bufferSize) {
return true;
}
auto remain = bufferSize - position;
if (remain > size) {
remain = size;
}
memcpy(data, static_cast<const uint8_t*>(buffer) + position, remain);
position += remain;
if (processedSize) {
*processedSize = remain;
}
return true;
}
private:
const uint8_t* buffer = nullptr;
size_t bufferSize = 0;
size_t position = 0;
};
class VectorOutStream : public SequentialOutStream {
public:
explicit VectorOutStream(std::vector<uint8_t>* buffer) : buffer(buffer) {
}
bool write(const void* data, size_t size) override {
auto oldSize = buffer->size();
buffer->resize(oldSize + size);
memcpy(&(*buffer)[oldSize], data, size);
return true;
}
private:
std::vector<uint8_t>* buffer;
};
class BuffPtrSeqOutStream : public SequentialOutStream {
public:
BuffPtrSeqOutStream(uint8_t* buffer, size_t size) : buffer(buffer), bufferSize(size) {
}
bool write(const void* data, size_t size) override {
auto remain = bufferSize - position;
if (remain > size) {
remain = size;
}
if (remain != 0) {
memcpy(buffer + position, data, remain);
position += remain;
}
return remain != 0 || size == 0;
}
private:
uint8_t* buffer = nullptr;
size_t bufferSize = 0;
size_t position = 0;
};
static const size_t kStreamStepSize = 1 << 31;
static SRes MyRead(const ISeqInStream* p, void* data, size_t* size) {
CSeqInStreamWrap* wrap = CONTAINER_FROM_VTBL(p, CSeqInStreamWrap, vt);
auto curSize = (*size < kStreamStepSize) ? *size : kStreamStepSize;
if (!wrap->inStream->read(data, curSize, &curSize)) {
return SZ_ERROR_READ;
}
*size = curSize;
return SZ_OK;
}
static size_t MyWrite(const ISeqOutStream* p, const void* buf, size_t size) {
auto* wrap = CONTAINER_FROM_VTBL(p, CSeqOutStreamWrap, vt);
if (wrap->outStream->write(buf, size)) {
return size;
}
return 0;
}
class Lzma2Encoder {
public:
Lzma2Encoder() {
encoder = Lzma2Enc_Create(&gAllocFuncs, &gAllocFuncs);
}
~Lzma2Encoder() {
Lzma2Enc_Destroy(encoder);
}
std::shared_ptr<Data> code(const std::shared_ptr<Data>& inputData) {
if (encoder == nullptr || inputData == nullptr || inputData->size() == 0) {
return nullptr;
}
auto inputSize = inputData->size();
CLzma2EncProps lzma2Props;
Lzma2EncProps_Init(&lzma2Props);
lzma2Props.lzmaProps.dictSize = inputSize;
lzma2Props.lzmaProps.level = 9;
lzma2Props.numTotalThreads = 4;
Lzma2Enc_SetProps(encoder, &lzma2Props);
std::vector<uint8_t> outBuf;
outBuf.resize(1 + 8);
outBuf[0] = Lzma2Enc_WriteProperties(encoder);
for (int i = 0; i < 8; i++) {
outBuf[1 + i] = static_cast<uint8_t>(inputSize >> (8 * i));
}
CSeqInStreamWrap inWrap = {};
inWrap.vt.Read = MyRead;
inWrap.inStream = std::make_unique<BuffPtrInStream>(
static_cast<const uint8_t*>(inputData->data()), inputSize);
CSeqOutStreamWrap outStream = {};
outStream.vt.Write = MyWrite;
outStream.outStream = std::make_unique<VectorOutStream>(&outBuf);
auto status =
Lzma2Enc_Encode2(encoder, &outStream.vt, nullptr, nullptr, &inWrap.vt, nullptr, 0, nullptr);
if (status != SZ_OK) {
return nullptr;
}
return Data::MakeWithCopy(&outBuf[0], outBuf.size());
}
private:
CLzma2EncHandle encoder = nullptr;
};
std::shared_ptr<Data> LzmaUtil::Compress(const std::shared_ptr<Data>& pixelData) {
Lzma2Encoder encoder;
return encoder.code(pixelData);
}
class Lzma2Decoder {
public:
Lzma2Decoder() {
decoder = Lzma2DecMt_Create(&gAllocFuncs, &gAllocFuncs);
}
~Lzma2Decoder() {
if (decoder) {
Lzma2DecMt_Destroy(decoder);
}
}
std::shared_ptr<Data> code(const std::shared_ptr<Data>& inputData) {
if (decoder == nullptr || inputData == nullptr || inputData->size() == 0) {
return nullptr;
}
auto input = static_cast<const uint8_t*>(inputData->data());
auto inputSize = inputData->size() - 9;
Byte prop = static_cast<const Byte*>(input)[0];
CLzma2DecMtProps props;
Lzma2DecMtProps_Init(&props);
props.inBufSize_ST = inputSize;
props.numThreads = 1;
UInt64 outBufferSize = 0;
for (int i = 0; i < 8; i++) {
outBufferSize |= (input[1 + i] << (i * 8));
}
auto outBuffer = new uint8_t[outBufferSize];
CSeqInStreamWrap inWrap = {};
inWrap.vt.Read = MyRead;
inWrap.inStream = std::make_unique<BuffPtrInStream>(input + 9, inputSize);
CSeqOutStreamWrap outWrap = {};
outWrap.vt.Write = MyWrite;
outWrap.outStream = std::make_unique<BuffPtrSeqOutStream>(outBuffer, outBufferSize);
UInt64 inProcessed = 0;
int isMT = false;
auto res = Lzma2DecMt_Decode(decoder, prop, &props, &outWrap.vt, &outBufferSize, 1, &inWrap.vt,
&inProcessed, &isMT, nullptr);
if (res == SZ_OK && inputSize == inProcessed) {
return Data::MakeAdopted(outBuffer, outBufferSize, Data::DeleteProc);
}
delete[] outBuffer;
return nullptr;
}
private:
CLzma2DecMtHandle decoder = nullptr;
};
std::shared_ptr<Data> LzmaUtil::Decompress(const std::shared_ptr<Data>& data) {
Lzma2Decoder decoder;
return decoder.code(data);
}
} // namespace pag
I am trying to create a function that uncompresses LZAM2 compressed data. I inspired myself from this tutorial which works great for LZMA and I tried to adapt it for LZMA2. I successfully created the compression function for LZMA2, but i have no success for the uncompression one.
Here is the compression function:
static void Compress2Inc(std::vector<unsigned char> &outBuf,
const std::vector<unsigned char> &inBuf)
{
CLzma2EncHandle enc = Lzma2Enc_Create(&SzAllocForLzma, &SzAllocForLzma2);
assert(enc);
CLzma2EncProps props;
Lzma2EncProps_Init(&props);
props.lzmaProps.writeEndMark = 1; // 0 or 1
SRes res = Lzma2Enc_SetProps(enc, &props);
assert(res == SZ_OK);
unsigned propsSize = LZMA_PROPS_SIZE;
outBuf.resize(propsSize);
res = Lzma2Enc_WriteProperties(enc);
//cout << res;
//assert(res == SZ_OK && propsSize == LZMA_PROPS_SIZE);
VectorInStream inStream = { &VectorInStream_Read, &inBuf, 0 };
VectorOutStream outStream = { &VectorOutStream_Write, &outBuf };
res = Lzma2Enc_Encode(enc,
(ISeqOutStream*)&outStream, (ISeqInStream*)&inStream,
0);
assert(res == SZ_OK);
Lzma2Enc_Destroy(enc);
}
Where:
static void *AllocForLzma2(void *, size_t size) { return BigAlloc(size); }
static void FreeForLzma2(void *, void *address) { BigFree(address); }
static ISzAlloc SzAllocForLzma2 = { AllocForLzma2, FreeForLzma2 };
static void *AllocForLzma(void *, size_t size) { return MyAlloc(size); }
static void FreeForLzma(void *, void *address) { MyFree(address); }
static ISzAlloc SzAllocForLzma = { AllocForLzma, FreeForLzma };
typedef struct
{
ISeqInStream SeqInStream;
const std::vector<unsigned char> *Buf;
unsigned BufPos;
} VectorInStream;
SRes VectorInStream_Read(void *p, void *buf, size_t *size)
{
VectorInStream *ctx = (VectorInStream*)p;
*size = min(*size, ctx->Buf->size() - ctx->BufPos);
if (*size)
memcpy(buf, &(*ctx->Buf)[ctx->BufPos], *size);
ctx->BufPos += *size;
return SZ_OK;
}
typedef struct
{
ISeqOutStream SeqOutStream;
std::vector<unsigned char> *Buf;
} VectorOutStream;
size_t VectorOutStream_Write(void *p, const void *buf, size_t size)
{
VectorOutStream *ctx = (VectorOutStream*)p;
if (size)
{
unsigned oldSize = ctx->Buf->size();
ctx->Buf->resize(oldSize + size);
memcpy(&(*ctx->Buf)[oldSize], buf, size);
}
return size;
}
Here is what I have so far with the uncompression function but Lzma2Dec_DecodeToBuf function returns error code 1(SZ_ERROR_DATA) and I just couldn't find anything on the web regarding this on the web.
static void Uncompress2Inc(std::vector<unsigned char> &outBuf,
const std::vector<unsigned char> &inBuf)
{
CLzma2Dec dec;
Lzma2Dec_Construct(&dec);
SRes res = Lzma2Dec_Allocate(&dec, outBuf.size(), &SzAllocForLzma);
assert(res == SZ_OK);
Lzma2Dec_Init(&dec);
outBuf.resize(UNCOMPRESSED_SIZE);
unsigned outPos = 0, inPos = LZMA_PROPS_SIZE;
ELzmaStatus status;
const unsigned BUF_SIZE = 10240;
while (outPos < outBuf.size())
{
unsigned destLen = min(BUF_SIZE, outBuf.size() - outPos);
unsigned srcLen = min(BUF_SIZE, inBuf.size() - inPos);
unsigned srcLenOld = srcLen, destLenOld = destLen;
res = Lzma2Dec_DecodeToBuf(&dec,
&outBuf[outPos], &destLen,
&inBuf[inPos], &srcLen,
(outPos + destLen == outBuf.size()) ? LZMA_FINISH_END : LZMA_FINISH_ANY,
&status);
assert(res == SZ_OK);
inPos += srcLen;
outPos += destLen;
if (status == LZMA_STATUS_FINISHED_WITH_MARK)
break;
}
Lzma2Dec_Free(&dec, &SzAllocForLzma);
outBuf.resize(outPos);
}
I am using Visual Studio 2008 and LZMA SDK downloaded from here. Someone here had the exact same problem, but i couldn't make use of his code...
Did anyone ever successfully uncompressed LZMA2 compressed files using LZMA SDK?
Please help!
A temporary workaround would be to replace SRes res = Lzma2Dec_Allocate(&dec, outBuf.size(), &SzAllocForLzma); with SRes res = Lzma2Dec_Allocate(&dec, 8, &SzAllocForLzma); in Uncompress2Inc function where 8 is a magic number...
However this is not the right way to solve the problem...
First mistake is that Lzma2Enc_WriteProperties doesn't return a result but a property byte which will have to be used as the second parameter of the Lzma2Dec_Allocate call in the Uncompress2Inc function. As a result we replace the magic number 8 with the property byte and everything works as expected.
In order to achieve this a 5 byte header must be added to the encoded data which will be extracted in the decoding function. Here is an example that works in VS2008(not the most perfect code but it works...I will get back later, when I have time, with a better example):
void Lzma2Benchmark::compressChunk(std::vector<unsigned char> &outBuf, const std::vector<unsigned char> &inBuf)
{
//! \todo This is a temporary workaround, size needs to be added to the
m_uncompressedSize = inBuf.size();
std::cout << "Uncompressed size is: " << inBuf.size() << std::endl;
DWORD tickCountBeforeCompression = GetTickCount();
CLzma2EncHandle enc = Lzma2Enc_Create(&m_szAllocForLzma, &m_szAllocForLzma2);
assert(enc);
CLzma2EncProps props;
Lzma2EncProps_Init(&props);
props.lzmaProps.writeEndMark = 1; // 0 or 1
props.lzmaProps.level = 9;
props.lzmaProps.numThreads = 3;
//props.numTotalThreads = 2;
SRes res = Lzma2Enc_SetProps(enc, &props);
assert(res == SZ_OK);
// LZMA_PROPS_SIZE == 5 bytes
unsigned propsSize = LZMA_PROPS_SIZE;
outBuf.resize(propsSize);
// I think Lzma2Enc_WriteProperties returns the encoding properties in 1 Byte
Byte properties = Lzma2Enc_WriteProperties(enc);
//! \todo This is a temporary workaround
m_propByte = properties;
//! \todo Here m_propByte and m_uncompressedSize need to be added to outBuf's 5 byte header so simply add those 2 values to outBuf and start the encoding from there.
BenchmarkUtils::VectorInStream inStream = { &BenchmarkUtils::VectorInStream_Read, &inBuf, 0 };
BenchmarkUtils::VectorOutStream outStream = { &BenchmarkUtils::VectorOutStream_Write, &outBuf };
res = Lzma2Enc_Encode(enc,
(ISeqOutStream*)&outStream,
(ISeqInStream*)&inStream,
0);
std::cout << "Compress time is: " << GetTickCount() - tickCountBeforeCompression << " milliseconds.\n";
assert(res == SZ_OK);
Lzma2Enc_Destroy(enc);
std::cout << "Compressed size is: " << outBuf.size() << std::endl;
}
void Lzma2Benchmark::unCompressChunk(std::vector<unsigned char> &outBuf, const std::vector<unsigned char> &inBuf)
{
DWORD tickCountBeforeUncompression = GetTickCount();
CLzma2Dec dec;
Lzma2Dec_Construct(&dec);
//! \todo Heere the property size and the uncompressed size need to be extracted from inBuf, which is the compressed data.
// The second parameter is a temporary workaround.
SRes res = Lzma2Dec_Allocate(&dec, m_propByte/*8*/, &m_szAllocForLzma);
assert(res == SZ_OK);
Lzma2Dec_Init(&dec);
outBuf.resize(m_uncompressedSize);
unsigned outPos = 0, inPos = LZMA_PROPS_SIZE;
ELzmaStatus status;
const unsigned BUF_SIZE = 10240;
while(outPos < outBuf.size())
{
SizeT destLen = std::min(BUF_SIZE, outBuf.size() - outPos);
SizeT srcLen = std::min(BUF_SIZE, inBuf.size() - inPos);
SizeT srcLenOld = srcLen, destLenOld = destLen;
res = Lzma2Dec_DecodeToBuf(&dec,
&outBuf[outPos],
&destLen,
&inBuf[inPos],
&srcLen,
(outPos + destLen == outBuf.size()) ? LZMA_FINISH_END : LZMA_FINISH_ANY,
&status);
assert(res == SZ_OK);
inPos += srcLen;
outPos += destLen;
if(status == LZMA_STATUS_FINISHED_WITH_MARK)
{
break;
}
}
Lzma2Dec_Free(&dec, &m_szAllocForLzma);
outBuf.resize(outPos);
std::cout << "Uncompress time is: " << GetTickCount() - tickCountBeforeUncompression << " milliseconds.\n";
}
I have a 2884765579 bytes file. This is double checked with this function, that returns that number:
size_t GetSize() {
const size_t current_position = mFile.tellg();
mFile.seekg(0, std::ios::end);
const size_t ret = mFile.tellg();
mFile.seekg(current_position);
return ret;
}
I then do:
mFile.seekg(pos, std::ios::beg);
// pos = 2883426827, which is < than the file size, 2884765579
This sets the failbit. errno is not changed. What steps can I take to troubleshoot this?
I am absolutely sure that:
The file size is really 2884765579
pos is really 2884765579
The failbit is not set before .seekg()
The failbit is set right after .seekg() and no other calls are made in between
The file is opened with the binary flag
EDIT: in case someone runs into the same problem.. Use this code I wrote (works on windows only) and many less headaches for you:
class BinaryIFile
{
public:
BinaryIFile(const string& path) : mPath(path), mFileSize(0) {
mFile = open(path.c_str(), O_RDONLY | O_BINARY);
if (mFile == -1)
FATAL(format("Cannot open %s: %s") % path.c_str() % strerror(errno));
}
~BinaryIFile() {
if (mFile != -1)
close(mFile);
}
string GetPath() const { return mPath; }
int64 GetSize() {
if (mFileSize)
return mFileSize;
const int64 current_position = _telli64(mFile);
_lseeki64(mFile, 0, SEEK_END);
mFileSize = _telli64(mFile);
_lseeki64(mFile, current_position, SEEK_SET);
return mFileSize;
}
int64 Read64() { return _Read<int64>(); }
int32 Read32() { return _Read<int32>(); }
int16 Read16() { return _Read<int16>(); }
int8 Read8() { return _Read<int8>(); }
float ReadFloat() { return _Read<float>(); }
double ReadDouble() { return _Read<double>(); }
void Skip(int64 bytes) { _lseeki64(mFile, bytes, SEEK_CUR); }
void Seek(int64 pos) { _lseeki64(mFile, pos, SEEK_SET); }
int64 Tell() { return _telli64(mFile); }
template <class T>
T Read() { return _Read<T>(); }
void Read(char *to, size_t size) {
const int ret = read(mFile, (void *)to, size);
if ((int)size != ret)
FATAL(format("Read error: attempted to read %d bytes, read() returned %d, errno: %s [we are at offset %d, file size is %d]") % size % ret % strerror(errno) % Tell() % GetSize());
}
template <class T>
BinaryIFile& operator>>(T& val) { val = _Read<T>(); return *this; }
private:
const string mPath;
int mFile;
int64 mFileSize;
template <class T>
T _Read() { T ret; if (sizeof(ret) != read(mFile, (void *)&ret, sizeof(ret))) FATAL("Read error"); return ret; }
};
You can seekg before a given position, so pos is signed. Try it with files of size 0x7fffffff and 0x80ffffff and see if the latter triggers the problem, that's my guess.