QDataStream on external memory? - c++

I'd like serialize QVector into char* array. I do this by the following code:
QVector<int> in;
...
QByteArray bytes;
QDataStream stream(&bytes, QIODevice::WriteOnly);
stream << in;
std::copy(bytes.constData(), bytes.constData() + bytes.size(), out);
I guarantee that out is large enough. Due to the fact that this code is called extremely often I would like to avoid this unnecessary std::copy operation and make either QByteArray or QDataStream work on preallocated user memory pointed by out. Is that possible? Any bight ideas?
UPDATE: QByteArray::fromRawData() doesn't match the needs cause it does not allow to change char* buffer it was created on, in other words, QByteArray performs deep copy on first modification of such created instance.
As they say. This ensures that the raw data array itself will never be modified by QByteArray.
SOLUTION: The solution proposed by #skyhisi does perfectly match my needs. The complete code is the following.
SimpleBuffer.hpp
#pragma once
#include <QtCore/QIODevice>
class SimpleBuffer : public QIODevice {
Q_OBJECT
Q_DISABLE_COPY(SimpleBuffer)
public:
SimpleBuffer(char* const begin, const char* const end) :
_begin(begin),
_end(end){}
virtual bool atEnd() const {
return _end == _begin;
}
virtual bool isSequential() const {
return true;
}
protected:
virtual qint64 readData(char*, qint64) {
return -1;
}
virtual qint64 writeData(const char* const data, const qint64 maxSize) {
const qint64 space = _end - _begin;
const qint64 toWrite = qMin(maxSize, space);
memcpy(_begin, data, size_t(toWrite));
_begin += toWrite;
return toWrite;
}
private:
char* _begin;
const char* const _end;
};
main.cpp
#include "SimpleBuffer.hpp"
#include <QtCore/QVector>
#include <QtCore/QDataStream>
#include <QtCore/QByteArray>
int main(int, char**) {
QVector<int> src;
src << 3 << 7 << 13 << 42 << 100500;
const size_t dataSize = sizeof(quint32) + src.size() * sizeof(int);
char* const data = new char[dataSize];
// prepare stream and write out the src vector
{
SimpleBuffer simpleBuffer(data, data + dataSize);
simpleBuffer.open(QIODevice::WriteOnly);
QDataStream os(&simpleBuffer);
os << src;
}
// read vector with QByteArray
QVector<int> dst;
{
const QByteArray byteArray = QByteArray::fromRawData((char*)data, dataSize);
QDataStream is(byteArray);
is >> dst;
}
delete [] data;
// check we've read exactly what we wrote
Q_ASSERT(src == dst);
return 0;
}

I think you may need to implement a QIODevice, you could make a very simple sequential device quite easily. Here's one I've quickly thrown together, I haven't checked it works (feel free to get it working and edit the post).
class SimpleBuffer : public QIODevice
{
Q_OBJECT
public:
SimpleBuffer(char* begin, char* end):mBegin(begin),mEnd(end){}
virtual bool atEnd() const {return mEnd == mBegin; }
virtual bool isSequential() const { return true; }
protected:
virtual qint64 readData(char*, qint64) { return -1; }
virtual qint64 writeData(const char* data, qint64 maxSize)
{
const qint64 space = mEnd - mBegin;
const qint64 toWrite = qMin(maxSize, space);
memcpy(mBegin, data, size_t(toWrite));
mBegin += toWrite;
return toWrite;
}
private:
char* mBegin;
char* mEnd;
Q_DISABLE_COPY(SimpleBuffer)
};

Maybe fromRawData works:
QByteArray QByteArray::fromRawData ( const char * data, int size ) [static]
Using it something like :
char* out=new char[enoughbytes]; // preallocate at suitable scope
QVector<int> in;
QByteArray ba=QByteArray::fromRawData(out,enoughbytes);
QDataStream stream(&ba,QIODevice::WriteOnly);
stream << in;
Note that QDataStream adds some of it's own data at the start of the data (not much though), so remember to preallocate a bit more for that, as well as for whatever additional data QVector serializes.

Why not use QBuffer?
QByteArray myBuffer;
myBuffer.reserve(10000); // no re-allocation
QBuffer buffer(&myBuffer);
buffer.open(QIODevice::WriteOnly);
QDataStream out(&buffer);
out << QApplication::palette();

Related

QString and QByteArray memory usage

The QByteArray documentation states that it is to be used when memory conservation is critical. I've been working on a class (FieldName) that keeps an internal store of commonly used QString's so that when two FieldName objects are compared for equality, a quick integer comparison is done.
My thinking was that I could save some memory and gain speed by using QByteArray instead of QString since I only need standard ASCII characters. But when I change out the QString in the following code for QByteArray, my tests show that speed is increased, but memory usage more than doubles. This runs counter to the documentation.
#ifndef H_FIELDNAME
#define H_FIELDNAME
#include <QString>
class FieldName
{
public:
FieldName() : mKey(0) { init(); }
FieldName(const QString& s);
const QString& constString() const;
bool operator==(const FieldName& other) const { return mKey == other.mKey; }
bool operator!=(const FieldName& other) const { return mKey != other.mKey; }
private:
int mKey;
void init();
int findKey(const QString& s);
};
#endif
CPP file:
#include "fieldname.h"
// #include <QMutex> // demo version not thread-safe
#include <QVector>
static QVector<QString*> FieldName__List;
static bool FieldName__FirstCall(true);
FieldName::FieldName(const QString& s)
{
init();
mKey = findKey(s);
}
const QString& FieldName::constString() const
{
return(*FieldName__List.at(mKey));
}
void FieldName::init()
{
if(FieldName__FirstCall)
{
if(FieldName__List.isEmpty())
{
FieldName__List.append(new QString(""));
FieldName__FirstCall = false;
}
}
}
int FieldName::findKey(const QString& s)
{
if(s.isEmpty())
return 0;
int sz(FieldName__List.size());
for(int idx=1; idx<sz; ++idx)
{
if(s == *FieldName_List.at(idx))
return idx;
}
FieldName__List.append(new QString(s));
return(FieldName__List.size() - 1);
}
Can anyone let me know what I may be doing that is causing this to use more memory when I substitute QByteArray for QString in the above class?
Testcode:
QStringList qsl; // filled with 84000 unique English words
FieldName fn;
foreach(QString s, qsl)
{
fn = FieldName(s);
// do stuff here
}

c++ compress with libzpaq to char* buffer

I have a std::vector<short> and would like to compress (and later decompress) with the libzpaq from https://github.com/zpaq/zpaq/ to something like char* buffer.
However I don't get the concept of this Reader and Writer class mentioned in the header file. How do I put my std::vector in to get a compressed buffer out?
Currently I have something like the following code.
#include <vector>
#include <string>
#include <stdio.h>
#include "libzpaq.h"
struct writer: public libzpaq::Writer {
void put(int c) {
}
};
struct reader: public libzpaq::Reader {
int get() {
}
};
void libzpaq::error(const char* msg) {
fprintf(stderr, "Oops: %s\n", msg);
exit(1);
}
int main() {
short a[] = {2,5,8,2,4,2,2,2,6,5,4,3,4,2,2};
std::vector<short> v(a, a+15);
char* buffer;
reader in;
writer out;
libzpaq::compress(&in, &out, "5");
}
And I wan't to compress the vector v into buffer. (And later decompress it again.)
But I don't understand the concept of the Reader and Writer struct/class.
The docu (http://mattmahoney.net/dc/libzpaq.3.html) also mentions the functions virtual int read(char* buf, int n) and virtual void write(const char* buf, int n) for the Reader and Writer. How can I cast a std::vector<short> to char* buf end get the length in n bytes of this buf?
Edit 1: I found a class StringBuffer in libzpaq.h line 1376. But something like
buffer = reinterpret_cast<char*> (&v[0]);
length = sizeof(short)*v.size();
libzpaq::StringBuffer inString, outString;
inString.read(buffer, length);
libzpaq::compress(&inString, &outString, "5");
std::cout << "size outstring: " << outString.size() << std::endl;
std::cout << "size instring: " << inString.size() << std::endl;
always gives me
size outstring: 0
size instring: 0
Even if I try it with a much larger vector v of some thousend random elements.
With Reader you provide byte by byte access to the data you want to compress. So with std::vector<short> it would look like this.
struct reader : public libzpaq::Reader {
reader(const std::vector<short>& v) :
m_v(v),
m_offset(0) {
}
int get() {
if (m_offset < m_v.size() * sizeof (short)) {
return *((char*) m_v.data() + m_offset++);
} else {
return -1;
}
}
int m_offset;
std::vector<short> m_v;
};
Writer should collect output data of the Reader. If you want to collect it in char array I could recommend to do it like this.
struct writer : public libzpaq::Writer {
void put(int c) {
m_buffer.push_back(c);
}
int size() {
m_buffer.size();
}
void copy_to(char* dst) {
memcpy(dst, m_buffer.data(), m_buffer.size());
}
std::vector<char> m_buffer;
};
Then call it:
writer w;
reader r(v1);
libzpaq::compress(&r, &w, "5");
char* buffer = new char[w.size()];
w.copy_to(buffer);
If you want to use StringBuffer then you should write some data to buffer, before read, that why it returns 0. Look at example:
char* buffer = reinterpret_cast<char*> (&v[0]);
int length = sizeof (short)*v.size();
libzpaq::StringBuffer in, out1, out2;
// fill buffer with source data
in.write(buffer, length);
// compress to out1
libzpaq::compress(&in, &out1, "5");
// decompress out1 to out2
libzpaq::decompress(&out1, &out2);
// check result
short* b = (short*)out2.data();
for(int i = 0; i < 15; ++i) {
std::cout << b[i] << std::endl;
}

Serializing object to byte-array in C++

I am working on an embedded device (microcontroller), and I want to save objects to permanent storage (an EEPROM). Most of the serialization solutions I can find, use the file-system in some way, but my target has no file-system.
Therefore my question is, how can I serialize an object to a byte-array so I can save that byte-array to an EEPROM afterwards?
Here is an example of what i am trying to do:
class Person{
//Constructor, getters and setters are omitted
void save(){
char buffer[sizeof(Person)];
serialize(buffer);
EEPROM::Save(buffer, sizeof(Person));
}
void load(){
char buffer[sizeof(Person)];
EEPROM::Load(buffer, sizeof(Person));
deserialize(buffer);
}
void serialize(char* result){
//?????
}
Person deserialize(char* buffer){
//??????
}
private:
char* name;
int age;
float weight;
};
It's likely that your code for save and load will be reasonably generic and would work best in a separate 'manager' class, leaving each data class only with the responsibility of rendering itself as re-loadable:
// Interface class
class Serializable
{
public:
virtual size_t serialize_size() const = 0;
virtual void serialize(char* dataOut) const = 0;
virtual void deserialize(const char* dataIn) = 0;
};
// Load / save manager
class EEPromManager
{
public:
void save( const Serializable& s )
{
char * data;
size_t data_len;
reserve_memory( data, data_len, s );
s.serialize( data );
EEPROM::Save( data , data_len );
delete [] data;
}
void load( Serializable& s )
{
char * data;
size_t data_len;
reserve_memory( data, data_len, s );
EEPROM::Load( data, data_len );
s.deserialize( data );
delete [] data;
}
private:
char* reserve_memory( char*& data, size_t& data_len, const Serializable& s )
{
return new char[ s.serialize_size() ];
}
};
Each class you intend to serialize / de-serialize should inherit from an interface which mandates the virtual interface for these functions. Note that you'll need to do your own memory management here. I've given a simple example but you'd probably want something a bit more robust.
Then each function should sequentially serialize all attributes of the class (chaining bases classes and calling serialize on aggregate objects if needed.)
class Person : public Serializable
{
public:
virtual size_t serialize_size() const
{
return SerializablePOD<char*>::serialize_size(name) +
SerializablePOD<int>::serialize_size(age) +
SerializablePOD<float>::serialize_size(weight);
}
virtual void serialize(char* dataOut) const
{
dataOut = SerializablePOD<char*>::serialize(dataOut, name);
dataOut = SerializablePOD<int>::serialize(dataOut, age);
dataOut = SerializablePOD<float>::serialize(dataOut, weight);
}
virtual void deserialize(const char* dataIn)
{
dataIn = SerializablePOD<char*>::deserialize(dataIn, name);
dataIn = SerializablePOD<int>::deserialize(dataIn, age);
dataIn = SerializablePOD<float>::deserialize(dataIn, weight);
}
private:
char* name;
int age;
float weight;
};
You'll benefit from generic code to serialize / de-serialize each separate type so you don't keep having code to write the length of strings etc. I.e. a serialize / de-serialize for each POD type:
template <typename POD>
class SerializablePOD
{
public:
static size_t serialize_size(POD str)
{
return sizeof(POD);
}
static char* serialize( char* target, POD value )
{
return memcpy( target, &value, serialize_size(value) );
}
static const char* deserialize( const char* source, POD& target )
{
memcpy( &target, source, serialize_size(target) );
return source + serialize_size(target);
}
};
template<>
size_t SerializablePOD<char*>::serialize_size(char* str)
{
return sizeof(size_t) + strlen(str);
}
template<>
const char* SerializablePOD<char*>::deserialize( const char* source, char*& target )
{
size_t length;
memcpy( &length, source, sizeof(size_t) );
memcpy( &target, source + sizeof(size_t), length );
return source + sizeof(size_t) + length;
}
Incidentally, you might also need to consider what will happen if you change the schema of an object in a software upgrade. Your saved objects would potentially become corrupted on reloading, unless you code round this using - for example - a class version identifier.
Final thought: At a micro level, what you're doing is in many ways similar to the way POD data is serialised for network transmission, so it may be that you can take advantage of libraries to do that - even if you don't have access to an operating system.
To save a string to binary, usually we save its length and then its content. To save other primitive data, we can simply store their binary form. So in your case, all you need to store is:
Length to name
char array of name
age
weight
So the code to serial is:
size_t buffer_size = sizeof(int) + strlen(name) + sizeof(age) + sizeof(weight);
char *buffer = new char[buffer_size];
*(int*)p = strlen(name); p += sizeof(int);
memcpy(p, name, strlen(name)); p += strlen(name);
*(int*)p = age; p += sizeof(int);
*(float*)p = weight;
EEPROM::Save(buffer, buffer_size);
delete[] buffer;
And to read a string from binary buffer, you read its length first, and then copy its data.

Get a char* from an ostream without copying

I have an ostream and data has been written to it. Now I want that data in the form of a char array. Is there a way to get the char buffer and its size without copying all of the bytes? I mean, I know I can use ostringstream and call str().c_str() on it but that produces a temporary copy.
I guess this is what you're looking for - a stream buffer that returns a pointer to its buffer:
#include <iostream>
#include <vector>
#include <string>
class raw_buffer : public std::streambuf
{
public:
raw_buffer(std::ostream& os, int buf_size = 256);
int_type overflow(int_type c) override;
std::streamsize showmanyc() override;
std::streamsize xsputn(const char_type*, std::streamsize) override;
int sync() override;
bool flush();
std::string const& str() const;
private:
std::ostream& os_;
std::vector<char> buffer;
std::string aux;
};
Now str() is simple. It returns a pointer to the underlying buffer of the auxillary buffer:
std::string const& raw_buffer::str() const
{
return aux;
}
The rest of the functions are the usual implementations for a stream buffer. showmanyc() should return the size of the auxiliary buffer (aux is just a running total of the entire buffer, buffer on the other hand is the size specified at construction).
For example, here is overflow(), which should update both buffers at same time but still treat buffer as the primary buffer:
raw_buffer::int_type raw_buffer::overflow(raw_buffer::int_type c) override
{
if (os_ && !traits_type::eq_int_type(c, traits_type::eof()))
{
aux += *this->pptr() = traits_type::to_char_type(c);
this->pbump(1);
if (flush())
{
this->pbump(-(this->pptr() - this->pbase()));
this->setp(this->buffer.data(),
this->buffer.data() + this->buffer.size());
return c;
}
}
return traits_type::eof();
}
flush() is used to copy the contents of buffer to the stream (os_), and sync() should be overrided to call flush() too.
xsputn also needs to be overrided to write to aux as well:
std::streamsize raw_buffer::xsputn(const raw_buffer::char_type* str, std::streamsize count) override
{
for (int i = 0; i < count; ++i)
{
if (traits_type::eq_int_type(this->sputc(str[i]), traits_type::eof()))
return i;
else
aux += str[i];
}
return count;
}
Now we can put this together with a customized stream:
class raw_ostream : private virtual raw_buffer
, public std::ostream
{
public:
raw_ostream(std::ostream& os) : raw_buffer(os)
, std::ostream(this)
{ }
std::string const& str() const
{
return this->raw_buffer::str();
}
std::streamsize count()
{
return this->str().size();
}
};
It can be used like this:
int main()
{
raw_ostream rostr(std::cout);
rostr << "Hello, World " << 123 << true << false;
auto& buf = rostr.str();
std::cout << buf;
}

Fix QString size (length)

For some reason I need to use string with fixed size. Now I'm looking on a QString class.
But there I have some questions with making QString object having constant size.
For example, I want to have string with size 10, it means, that if I would try to write some string with more than 100 chars in it, it would cut all the characters after 100 one.
I found constructor for QString in Qt docs, but I'm not sure if it would work as I told
QString( int size , QChar ch)
What can you suggest in such situation?
You can have a wrapper class that has a string, but is not a string, yet it can be used wherever a QString could be used. It can also be used with all QString's methods and operators, as long as you treat it like it were a pointer.
#include <QString>
class FixedWidthString {
mutable QString m_string;
//! Ignored if negative.
int m_maxLength;
inline const QString& data() const {
if (m_maxLength >= 0 && m_string.length() > m_maxLength)
m_string.truncate(m_maxLength);
return m_string;
}
inline QString& data() {
if (m_maxLength >= 0 && m_string.length() > m_maxLength)
m_string.truncate(m_maxLength);
return m_string;
}
public:
explicit FixedWidthString(int maxLength = -1) : m_maxLength(maxLength) {}
explicit FixedWidthString(const QString & str, int maxLength = -1) : m_string(str), m_maxLength(maxLength) {}
operator const QString&() const { return data(); }
operator QString&() { return data(); }
QString* operator->() { return &data(); }
const QString* operator->() const { return &data(); }
QString& operator*() { return data(); }
const QString& operator*() const { return data(); }
FixedWidthString & operator=(const FixedWidthString& other) {
m_string = *other;
return *this;
}
};
int main() {
FixedWidthString fs(3);
FixedWidthString fs2(2);
*fs = "FooBarBaz";
Q_ASSERT(*fs == "Foo");
fs->truncate(2);
Q_ASSERT(*fs == "Fo");
fs->append("Roo");
Q_ASSERT(*fs == "FoR");
fs->truncate(1);
*fs += "abc";
Q_ASSERT(*fs == "Fab");
fs2 = fs;
Q_ASSERT(*fs2 == "Fa");
}