I would like to load a wxXmlDocument from a std::istream but unfortunately, there's no Load(std::istream&) member function, even if wxWidget is compiled using standard input/output streams.
For what it's worth, I'm using wxWidgets 3.1.0 on macOS.
I don't know if there are alternatives but, since wxXmlDocument provides a Load(wxInputStream&), a solution can be to define an adapter like this one:
class myStdInputStreamAdapter : public wxInputStream {
public:
myStdInputStreamAdapter(std::istream &s): stream{s} {}
protected:
std::istream &stream;
virtual size_t OnSysRead(void *buffer, size_t bufsize) {
std::streamsize size = 0;
stream.peek();
if (stream.fail() || stream.bad()) {
m_lasterror = wxSTREAM_READ_ERROR;
} else if (stream.eof()) {
m_lasterror = wxSTREAM_EOF;
} else {
size = stream.readsome(static_cast<std::istream::char_type *>(buffer),
bufsize);
}
return size;
}
};
And then use it to load the xml:
void f(std::istream &istream) {
wxXmlDocument xml;
myStdInputStreamAdapter inputStreamAdapter(istream);
xml.Load(inputStreamAdapter);
}
Related
Following is the function which I want to unit test:
void sampleFunc()
{
FILE *file = fopen(path, "rb");
if (!file) {
std::cout << "File couldn't be opened!" << std::endl;
}
const int bufSize = 32768;
unsigned char *buffer = (unsigned char*) malloc(bufSize);
if (!buffer) {
fclose(file);
std::cout << "Failed to allocate buffer for SHA256 computation." << std::endl;
}
// ..
// Read file into buffer
// ..
}
As shown in the code, my function uses many standard library functions. So am I supposed to mock standard library functions or make actual calls to functions?
What you might do to test your function:
class IFileManager
{
public:
virtual ~IFileManager() = default;
// You could even improve interface by returning RAII object
virtual FILE* fopen(const char* path, const char* mode) = 0;
virtual void fclose(FILE*) = 0;
// ...
};
class FileManager : public IFileManager
{
public:
FILE* fopen(const char* path, const char* mode) override { return ::fopen(path, mode); }
int fclose(FILE* f) override { return ::fclose(f); }
// ...
};
class IAllocator
{
public:
virtual ~IAllocator() = default;
virtual void* allocate(std::size_t) = 0;
virtual void deallocate(void*) = 0;
};
class Allocator : public IAllocator
{
public:
void* allocate(std::size_t size) override { return malloc(size); }
void deallocate(void* p) override { free(p); }
};
Your function becomes:
void sampleFunc(IFileManager& fileManager, IAllocator& allocator)
{
FILE *file = fileManager.fopen(path, "rb");
if(!file) {
std::cout << "File couldn't be opened!" << endl;
return;
}
const int bufSize = 32768;
unsigned char *buffer = (unsigned char*) allocator.allocate(bufSize);
if(!buffer) {
fileManager.fclose(file);
std::cout << "Failed to allocate buffer for SHA256 computation." << std::endl;
return;
}
// Read file into buffer
// ...
}
Finally, you can easily mock IFileManager and IAllocator.
Without that interface, you would have to make standard function behave as you expect, which is not necessary simple feasible (wrong path, limit memory (which limit))
Care also that implementation might have more limitations. that the one from interface (MAX_PATH, non-regular files, UNC path)
I am using qDebug(), qInfo() and so on in combination of qInstallMessageHandler to write my logfiles. I also get an output on my batch screen, when executing my application.
I only found QT_NO_DEBUG_OUTPUT, but I want to toggle this configuration while run-time. Is there a way to prevent Qt from writing to std output?
sadly no, you only get access to the messages, but cannot prevent from beeing written to std output.
That's false in Qt 5 at the very least. The message printing code looks as follows: you can clearly see that only the message handler is being used:
if (grabMessageHandler()) {
// prefer new message handler over the old one
if (msgHandler.load() == qDefaultMsgHandler
|| messageHandler.load() != qDefaultMessageHandler) {
(*messageHandler.load())(msgType, context, message);
} else {
(*msgHandler.load())(msgType, message.toLocal8Bit().constData());
}
ungrabMessageHandler();
} else {
fprintf(stderr, "%s\n", message.toLocal8Bit().constData());
}
As you can see, the fprintf(stderr, ...) is a fallback only if recursion is detected from within the messageHandler itself.
Thus, all you need to prevent any debug output is to implement and set your own messageHandler.
To completely turn off all the debug output in Qt, the following works:
#include <QtCore>
int main() {
qDebug() << "I'm not quiet at all yet";
qInstallMessageHandler(+[](QtMsgType, const QMessageLogContext &, const QString &){});
qDebug() << "I'm very, very quiet";
}
In any case, a sensible implementation of an application-wide file logger might look as follows - it doesn't recreate the QTextStream unnecessarily; it uses QString::toUtf8() instead, and explicitly writes line endings.
#include <QtCore>
class Logger {
static struct Data {
Logger *instance;
QtMessageHandler chainedHandler;
} d;
bool m_isOpen;
QFile m_logFile;
QtMessageHandler m_oldHandler = {};
static void handler(QtMsgType type, const QMessageLogContext &context, const QString &msg) {
if (d.instance)
d.instance->log(msg);
if (d.chainedHandler)
d.chainedHandler(type, context, msg);
}
public:
enum ChainMode { DontChain, Chain };
Logger() {
Q_ASSERT(!instance());
m_logFile.setFileName("myLog.txt");
m_isOpen = m_logFile.open(QIODevice::WriteOnly | QIODevice::Append | QIODevice::Text);
d.instance = this;
}
~Logger() { uninstallHandler(); }
bool isOpen() const { return m_isOpen; }
void installHandler(ChainMode mode) {
Q_ASSERT(!m_oldHandler);
m_oldHandler = qInstallMessageHandler(handler);
if (mode == Chain)
d.chainedHandler = m_oldHandler;
}
void uninstallHandler() {
if (m_oldHandler) {
m_oldHandler = nullptr;
d.chainedHandler = nullptr;
qInstallMessageHandler(m_oldHandler);
}
}
/// This method is *not* thread-safe. Use with a thread-safe wrapper such as `qDebug`.
void log(const QString & msg) {
if (isOpen()) {
m_logFile.write(msg.toUtf8());
m_logFile.write("\n", 1);
}
}
/// Closes the log file early - this is mostly used for testing.
void endLog() {
uninstallHandler();
m_logFile.close();
}
static Logger *instance() { return d.instance; }
};
Logger::Data Logger::d;
template <typename T> QByteArray streamOutputFor(const T &data) {
QBuffer buf;
buf.open(QIODevice::ReadWrite | QIODevice::Text);
QTextStream s(&buf);
s << data << endl;
buf.close();
return buf.data();
}
QByteArray readEnd(const QString &fileName, int count) {
QFile file(fileName);
if (file.open(QIODevice::ReadOnly | QIODevice::Text) && file.size() >= count) {
file.seek(file.size() - count);
return file.readAll();
}
return {};
}
void test() {
auto const entry = QDateTime::currentDateTime().toString().toUtf8();
Q_ASSERT(Logger::instance()->isOpen());
qDebug() << entry.data();
Logger::instance()->endLog();
auto reference = streamOutputFor(entry.data());
auto readback = readEnd("myLog.txt", reference.size());
Q_ASSERT(!reference.isEmpty());
Q_ASSERT(readback == reference);
}
int main() {
Logger logger;
logger.installHandler(Logger::DontChain);
qDebug() << "Something or else";
test();
}
By accident I found out that I can use read and write on socket descriptors. Can I somehow (ab)use the fstream mechanism to output data into the socket descriptor?
The standard file stream doesn't support use of a file descriptor. However, the I/O stream classes make it reasonably easy to create your own abstraction which allows creating your own sources of or destination for characters. The magic class is std::streambuf whose responsibility is to buffer characters and read or write characters at appropriate times. Nicolai Josuttis's "The C++ Standard Library" has a detailed description of how to do so (the basis of which I contributed to Nico many years ago). A simple implementation of a stream buffer using a socket for reading and writing would look something like this:
#include <algorithm>
#include <iostream>
#include <iterator>
#include <streambuf>
#include <cstddef>
#include <unistd.h>
class fdbuf
: public std::streambuf
{
private:
enum { bufsize = 1024 };
char outbuf_[bufsize];
char inbuf_[bufsize + 16 - sizeof(int)];
int fd_;
public:
typedef std::streambuf::traits_type traits_type;
fdbuf(int fd);
~fdbuf();
void open(int fd);
void close();
protected:
int overflow(int c);
int underflow();
int sync();
};
fdbuf::fdbuf(int fd)
: fd_(-1) {
this->open(fd);
}
fdbuf::~fdbuf() {
this->close();
}
void fdbuf::open(int fd) {
this->close();
this->fd_ = fd;
this->setg(this->inbuf_, this->inbuf_, this->inbuf_);
this->setp(this->outbuf_, this->outbuf_ + bufsize - 1);
}
void fdbuf::close() {
if (!(this->fd_ < 0)) {
this->sync();
::close(this->fd_);
}
}
int fdbuf::overflow(int c) {
if (!traits_type::eq_int_type(c, traits_type::eof())) {
*this->pptr() = traits_type::to_char_type(c);
this->pbump(1);
}
return this->sync() == -1
? traits_type::eof()
: traits_type::not_eof(c);
}
int fdbuf::sync() {
if (this->pbase() != this->pptr()) {
std::streamsize size(this->pptr() - this->pbase());
std::streamsize done(::write(this->fd_, this->outbuf_, size));
// The code below assumes that it is success if the stream made
// some progress. Depending on the needs it may be more
// reasonable to consider it a success only if it managed to
// write the entire buffer and, e.g., loop a couple of times
// to try achieving this success.
if (0 < done) {
std::copy(this->pbase() + done, this->pptr(), this->pbase());
this->setp(this->pbase(), this->epptr());
this->pbump(size - done);
}
}
return this->pptr() != this->epptr()? 0: -1;
}
int fdbuf::underflow()
{
if (this->gptr() == this->egptr()) {
std::streamsize pback(std::min(this->gptr() - this->eback(),
std::ptrdiff_t(16 - sizeof(int))));
std::copy(this->egptr() - pback, this->egptr(), this->eback());
int done(::read(this->fd_, this->eback() + pback, bufsize));
this->setg(this->eback(),
this->eback() + pback,
this->eback() + pback + std::max(0, done));
}
return this->gptr() == this->egptr()
? traits_type::eof()
: traits_type::to_int_type(*this->gptr());
}
int main()
{
fdbuf inbuf(0);
std::istream in(&inbuf);
fdbuf outbuf(1);
std::ostream out(&outbuf);
std::copy(std::istreambuf_iterator<char>(in),
std::istreambuf_iterator<char>(),
std::ostreambuf_iterator<char>(out));
}
Say I have a function that creates a QIODevice (e.g. a QFile), then returns a pointer to a QDataStream constructed from the QIODevice. What is the best way to deal with the memory allocation here? Clearly the QIODevice must be heap allocated to stay available to the QDataStream upon function termination, however the destruction of a QDataStream does not destroy or close the device. Is there a standard way of dealing with this seemingly common problem?
Ideally I want a function that returns an object (not a pointer to an object) that behaves like a QDataStream but upon destruction closes the device. Effectively a standard library input stream.
Example code:
QDataStream* getStream(const QString& filename) {
QFile* file = new QFile(filename); // needs to be explicitly deleted later
file->open(QIODevice::ReadOnly);
QDataStream* out = new QDataStream(&file); // same here
return out;
}
std::shared_ptr<QDataStream> getStream(const QString& filename)
{
QFile* file = new QFile(filename); // needs to be explicitly deleted later
file->open(QIODevice::ReadOnly);
std:shared_ptr<QDataStream> out(new QDataStream(&file), QDSDeleter);
return out;
}
void QDSDeleter(QDataStream* s)
{
QIODevice* device = s->device();
device->close();
delete device;
}
std::unique_ptr is another option depending on your needs; here's a reference for the former if you need it.
Edit: Qt also has the facility for this with its QSharedPointer class, where you can also supply a deleter as a constructor argument. Other pointer wrapper options are given there. Thanks #RA. for the correction.
QDataStream has a handy owndev private member that you can set to true to have the stream effectively own the device. The stream can also be easily made moveable - bringing it very close to your requirement of it behaving like a value. Ideally, you would modify your copy of Qt to implement it, but it can also be worked around.
// https://github.com/KubaO/stackoverflown/tree/master/questions/qdatastream-move-own-13039614
#include <QDataStream>
class DataStream : public QDataStream {
struct Proxy {
QScopedPointer<QDataStreamPrivate> d;
QIODevice *dev;
bool owndev;
bool noswap;
QDataStream::ByteOrder byteorder;
int ver;
QDataStream::Status q_status;
#if QT_VERSION < QT_VERSION_CHECK(5, 0, 0)
virtual ~Proxy();
#endif
};
static Proxy *p(QDataStream *ds) { return reinterpret_cast<Proxy *>(ds); }
static const Proxy *p(const QDataStream *ds) {
return reinterpret_cast<const Proxy *>(ds);
}
#if defined(QT_TESTLIB_LIB) || defined(QT_MODULE_TEST)
friend class DataStreamTest;
#endif
public:
DataStream() = default;
using QDataStream::QDataStream;
DataStream(DataStream &&other) : DataStream(static_cast<QDataStream &&>(other)) {}
DataStream(QDataStream &&other) {
using std::swap;
Proxy &o = *p(&other);
Proxy &t = *p(this);
swap(t.d, o.d);
swap(t.dev, o.dev);
swap(t.owndev, o.owndev);
swap(t.noswap, o.noswap);
swap(t.byteorder, o.byteorder);
swap(t.ver, o.ver);
swap(t.q_status, o.q_status);
}
DataStream &operator=(DataStream &&other) {
return *this = static_cast<QDataStream &&>(other);
}
DataStream &operator=(QDataStream &&other) {
this->~DataStream();
new (this) DataStream(std::move(other));
return *this;
}
void setOwnedDevice(QIODevice *dev) {
setDevice(dev);
p(this)->owndev = true;
}
bool ownsDevice() const { return p(this)->owndev; }
static bool ownsDevice(const QDataStream *ds) { return p(ds)->owndev; }
};
Since QObject has a built-in reference count, we could also have the QDataStream act as a shared pointer for it if we wished to.
The following requirements are tested on both Qt 4.8 and 5.10:
PASS : DataStreamTest::isBinaryCompatible()
PASS : DataStreamTest::streams()
PASS : DataStreamTest::movesFromNotOwnedQDataStream()
PASS : DataStreamTest::movesFromNotOwnedDataStream()
PASS : DataStreamTest::assignsFromNotOwnedQDataStream()
PASS : DataStreamTest::assignsFromNotOwnedDataStream()
PASS : DataStreamTest::returnsFromNotOwnedQDataStream()
PASS : DataStreamTest::returnsFromNotOwnedDataStream()
PASS : DataStreamTest::movesFromOwnedQDataStream()
PASS : DataStreamTest::moveFromOwnedDataStream()
PASS : DataStreamTest::assignsFromOwnedQDataStream()
PASS : DataStreamTest::assignsFromOwnedDataStream()
PASS : DataStreamTest::returnsFromOwnedQDataStream()
PASS : DataStreamTest::returnsFromOwnedDataStream()
The test suite follows. The binary compatibility test is extensive and all but excludes the possibility that the UB we depend on is problematic. Note that QDataStream's layout cannot change within the major Qt version - so the above code will work on all future Qt 5 versions.
#include <QtTest>
class DataStreamTest : public QObject {
Q_OBJECT
static QObjectData *getD(QObject *obj) {
return static_cast<DataStreamTest *>(obj)->d_ptr.data();
}
static bool wasDeleted(QObject *obj) { return getD(obj)->wasDeleted; }
template <typename T, typename... Args>
DataStream make_stream(Args &&... args) {
return T(std::forward<Args>(args)...);
}
static QDataStream::ByteOrder flipped(QDataStream::ByteOrder o) {
return (o == QDataStream::BigEndian) ? QDataStream::LittleEndian
: QDataStream::BigEndian;
}
Q_SLOT void isBinaryCompatible() {
QCOMPARE(sizeof(DataStream), sizeof(QDataStream));
QCOMPARE(sizeof(DataStream::Proxy), sizeof(QDataStream));
struct Test {
QByteArray data;
QDataStream ds{&data, QIODevice::ReadWrite};
void check(int loc = 0) {
if (!loc) {
check(1);
ds.setDevice(nullptr);
check(1);
}
QCOMPARE(!!ds.device(), DataStream::ownsDevice(&ds));
QCOMPARE(ds.device(), DataStream::p(&ds)->dev);
if (!loc) check(2);
bool noswap = DataStream::p(&ds)->noswap;
QCOMPARE(noswap, DataStream::p(&ds)->noswap);
QCOMPARE(ds.byteOrder(), DataStream::p(&ds)->byteorder);
if (loc != 2) {
ds.setByteOrder(flipped(ds.byteOrder()));
noswap = !noswap;
}
if (!loc) check(2);
QCOMPARE(noswap, DataStream::p(&ds)->noswap);
if (!loc) check(3);
QCOMPARE(ds.version(), DataStream::p(&ds)->ver);
if (loc != 3) ds.setVersion(QDataStream::Qt_4_0);
if (!loc) check(3);
if (!loc) check(4);
QCOMPARE(ds.status(), DataStream::p(&ds)->q_status);
if (loc != 4) ds.setStatus(QDataStream::ReadPastEnd);
if (!loc) check(4);
}
} test;
test.check();
}
Q_SLOT void streams() {
QString str{"Hello, world"};
QVector<uint> ints{44, 0xDEADBEEF, 1};
QByteArray data;
DataStream ds(&data, QIODevice::ReadWrite);
ds << str << ints;
ds.device()->reset();
QString str2;
QVector<uint> ints2;
ds >> str2 >> ints2;
QCOMPARE(str2, str);
QCOMPARE(ints2, ints);
}
Q_SLOT void movesFromNotOwnedQDataStream() {
QBuffer buf;
QDataStream ds(&buf);
QVERIFY(ds.device() == &buf);
DataStream ds2(std::move(ds));
QVERIFY(!ds.device());
QVERIFY(ds2.device() == &buf);
QVERIFY(!wasDeleted(&buf));
}
Q_SLOT void movesFromNotOwnedDataStream() {
QBuffer buf;
DataStream ds(&buf);
QVERIFY(ds.device() == &buf);
DataStream ds2(std::move(ds));
QVERIFY(!ds.device());
QVERIFY(ds2.device() == &buf);
QVERIFY(!wasDeleted(&buf));
}
Q_SLOT void assignsFromNotOwnedQDataStream() {
QBuffer buf;
QDataStream ds(&buf);
QVERIFY(ds.device() == &buf);
DataStream ds2;
ds2 = std::move(ds);
QVERIFY(!ds.device());
QVERIFY(ds2.device() == &buf);
QVERIFY(!wasDeleted(&buf));
}
Q_SLOT void assignsFromNotOwnedDataStream() {
QBuffer buf;
DataStream ds(&buf);
QVERIFY(ds.device() == &buf);
DataStream ds2;
ds2 = std::move(ds);
QVERIFY(!ds.device());
QVERIFY(ds2.device() == &buf);
QVERIFY(!wasDeleted(&buf));
}
Q_SLOT void returnsFromNotOwnedQDataStream() {
QBuffer buf;
{
auto ds = make_stream<QDataStream>(&buf);
QVERIFY(ds.device());
QVERIFY(!ds.ownsDevice());
}
QVERIFY(!wasDeleted(&buf));
}
Q_SLOT void returnsFromNotOwnedDataStream() {
QBuffer buf;
buf.open(QIODevice::ReadWrite);
{
auto ds = make_stream<DataStream>(&buf);
QVERIFY(ds.device());
QVERIFY(!ds.ownsDevice());
}
QVERIFY(!wasDeleted(&buf));
}
Q_SLOT void movesFromOwnedQDataStream() {
QPointer<QIODevice> buf;
{
QByteArray data;
QDataStream ds(&data, QIODevice::ReadWrite);
QVERIFY(DataStream::ownsDevice(&ds));
buf = ds.device();
DataStream ds2(std::move(ds));
QVERIFY(!ds.device());
QVERIFY(ds2.device() == buf);
QVERIFY(buf);
}
QVERIFY(!buf);
}
Q_SLOT void moveFromOwnedDataStream() {
QPointer<QBuffer> buf(new QBuffer);
{
DataStream ds;
ds.setOwnedDevice(buf);
QVERIFY(ds.device() == buf);
DataStream ds2(std::move(ds));
QVERIFY(!ds.device());
QVERIFY(ds2.device() == buf);
QVERIFY(buf);
}
QVERIFY(!buf);
}
Q_SLOT void assignsFromOwnedQDataStream() {
QPointer<QIODevice> buf;
{
QByteArray data;
QDataStream ds(&data, QIODevice::ReadWrite);
QVERIFY(DataStream::ownsDevice(&ds));
buf = ds.device();
DataStream ds2;
ds2 = std::move(ds);
QVERIFY(!ds.device());
QVERIFY(ds2.device() == buf);
QVERIFY(buf);
}
QVERIFY(!buf);
}
Q_SLOT void assignsFromOwnedDataStream() {
QPointer<QBuffer> buf(new QBuffer);
{
DataStream ds;
ds.setOwnedDevice(buf);
QVERIFY(ds.device() == buf);
DataStream ds2;
ds2 = std::move(ds);
QVERIFY(!ds.device());
QVERIFY(ds2.device() == buf);
QVERIFY(buf);
}
QVERIFY(!buf);
}
Q_SLOT void returnsFromOwnedQDataStream() {
QPointer<QIODevice> dev;
QByteArray data;
{
auto ds = make_stream<QDataStream>(&data, QIODevice::ReadWrite);
dev = ds.device();
QVERIFY(ds.device());
QVERIFY(ds.ownsDevice());
}
QVERIFY(!dev);
}
Q_SLOT void returnsFromOwnedDataStream() {
QPointer<QIODevice> dev;
QByteArray data;
{
auto ds = make_stream<DataStream>(&data, QIODevice::ReadWrite);
dev = ds.device();
QVERIFY(ds.device());
QVERIFY(ds.ownsDevice());
}
QVERIFY(!dev);
}
};
QTEST_MAIN(DataStreamTest)
#include "main.moc"
The following code shows unexpected behaviour on my machine (tested with Visual C++ 2008 SP1 on Windows XP and VS 2012 on Windows 7):
#include <iostream>
#include "Windows.h"
int main() {
SetConsoleOutputCP( CP_UTF8 );
std::cout << "\xc3\xbc";
int fail = std::cout.fail() ? '1': '0';
fputc( fail, stdout );
fputs( "\xc3\xbc", stdout );
}
I simply compiled with cl /EHsc test.cpp.
Windows XP: Output in a console window is
ü0ü (translated to Codepage 1252, originally shows some line drawing
charachters in the default Codepage, perhaps 437). When I change the settings
of the console window to use the "Lucida Console" character set and run my
test.exe again, output is changed to 1ü, which means
the character ü can be written using fputs and its UTF-8 encoding C3 BC
std::cout does not work for whatever reason
the streams failbit is setting after trying to write the character
Windows 7: Output using Consolas is ��0ü. Even more interesting. The correct bytes are written, probably (at least when redirecting the output to a file) and the stream state is ok, but the two bytes are written as separate characters).
I tried to raise this issue on "Microsoft Connect" (see here),
but MS has not been very helpful. You might as well look here
as something similar has been asked before.
Can you reproduce this problem?
What am I doing wrong? Shouldn't the std::cout and the fputs have the same
effect?
SOLVED: (sort of) Following mike.dld's idea I implemented a std::stringbuf doing the conversion from UTF-8 to Windows-1252 in sync() and replaced the streambuf of std::cout with this converter (see my comment on mike.dld's answer).
I understand the question is quite old, but if someone would still be interested, below is my solution. I've implemented a quite simple std::streambuf descendant and then passed it to each of standard streams on the very beginning of program execution.
This allows you to use UTF-8 everywhere in your program. On input, data is taken from console in Unicode and then converted and returned to you in UTF-8. On output the opposite is done, taking data from you in UTF-8, converting it to Unicode and sending to console. No issues found so far.
Also note, that this solution doesn't require any codepage modification, with either SetConsoleCP, SetConsoleOutputCP or chcp, or something else.
That's the stream buffer:
class ConsoleStreamBufWin32 : public std::streambuf
{
public:
ConsoleStreamBufWin32(DWORD handleId, bool isInput);
protected:
// std::basic_streambuf
virtual std::streambuf* setbuf(char_type* s, std::streamsize n);
virtual int sync();
virtual int_type underflow();
virtual int_type overflow(int_type c = traits_type::eof());
private:
HANDLE const m_handle;
bool const m_isInput;
std::string m_buffer;
};
ConsoleStreamBufWin32::ConsoleStreamBufWin32(DWORD handleId, bool isInput) :
m_handle(::GetStdHandle(handleId)),
m_isInput(isInput),
m_buffer()
{
if (m_isInput)
{
setg(0, 0, 0);
}
}
std::streambuf* ConsoleStreamBufWin32::setbuf(char_type* /*s*/, std::streamsize /*n*/)
{
return 0;
}
int ConsoleStreamBufWin32::sync()
{
if (m_isInput)
{
::FlushConsoleInputBuffer(m_handle);
setg(0, 0, 0);
}
else
{
if (m_buffer.empty())
{
return 0;
}
std::wstring const wideBuffer = utf8_to_wstring(m_buffer);
DWORD writtenSize;
::WriteConsoleW(m_handle, wideBuffer.c_str(), wideBuffer.size(), &writtenSize, NULL);
}
m_buffer.clear();
return 0;
}
ConsoleStreamBufWin32::int_type ConsoleStreamBufWin32::underflow()
{
if (!m_isInput)
{
return traits_type::eof();
}
if (gptr() >= egptr())
{
wchar_t wideBuffer[128];
DWORD readSize;
if (!::ReadConsoleW(m_handle, wideBuffer, ARRAYSIZE(wideBuffer) - 1, &readSize, NULL))
{
return traits_type::eof();
}
wideBuffer[readSize] = L'\0';
m_buffer = wstring_to_utf8(wideBuffer);
setg(&m_buffer[0], &m_buffer[0], &m_buffer[0] + m_buffer.size());
if (gptr() >= egptr())
{
return traits_type::eof();
}
}
return sgetc();
}
ConsoleStreamBufWin32::int_type ConsoleStreamBufWin32::overflow(int_type c)
{
if (m_isInput)
{
return traits_type::eof();
}
m_buffer += traits_type::to_char_type(c);
return traits_type::not_eof(c);
}
The usage then is as follows:
template<typename StreamT>
inline void FixStdStream(DWORD handleId, bool isInput, StreamT& stream)
{
if (::GetFileType(::GetStdHandle(handleId)) == FILE_TYPE_CHAR)
{
stream.rdbuf(new ConsoleStreamBufWin32(handleId, isInput));
}
}
// ...
int main()
{
FixStdStream(STD_INPUT_HANDLE, true, std::cin);
FixStdStream(STD_OUTPUT_HANDLE, false, std::cout);
FixStdStream(STD_ERROR_HANDLE, false, std::cerr);
// ...
std::cout << "\xc3\xbc" << std::endl;
// ...
}
Left out wstring_to_utf8 and utf8_to_wstring could easily be implemented with WideCharToMultiByte and MultiByteToWideChar WinAPI functions.
Oi. Congratulations on finding a way to change the code page of the console from inside your program. I didn't know about that call, I always had to use chcp.
I'm guessing the C++ default locale is getting involved. By default it will use the code page provide by GetThreadLocale() to determine the text encoding of non-wstring stuff. This generally defaults to CP1252. You could try using SetThreadLocale() to get to UTF-8 (if it even does that, can't recall), with the hope that std::locale defaults to something that can handle your UTF-8 encoding.
It's time to close this now. Stephan T. Lavavej says the behaviour is "by design", although I cannot follow this explanation.
My current knowledge is: Windows XP console in UTF-8 codepage does not work with C++ iostreams.
Windows XP is getting out of fashion now and so does VS 2008. I'd be interested to hear if the problem still exists on newer Windows systems.
On Windows 7 the effect is probably due to the way the C++ streams output characters. As seen in an answer to Properly print utf8 characters in windows console, UTF-8 output fails with C stdio when printing one byte after after another like putc('\xc3'); putc('\xbc'); as well. Perhaps this is what C++ streams do here.
I just follow mike.dld's answer in this question, and add the printf support for the UTF-8 string.
As mkluwe mentioned in his answer that by default, printf function will output to the console one by one byte, while the console can't handle single byte correctly. My method is quite simple, I use the snprintf function to print the whole content to a internal string buffer, and then dump the buffer to std::cout.
Here is the full testing code:
#include <iostream>
#include <locale>
#include <windows.h>
#include <cstdlib>
using namespace std;
// https://stackoverflow.com/questions/4358870/convert-wstring-to-string-encoded-in-utf-8
#include <codecvt>
#include <string>
// convert UTF-8 string to wstring
std::wstring utf8_to_wstring (const std::string& str)
{
std::wstring_convert<std::codecvt_utf8<wchar_t>> myconv;
return myconv.from_bytes(str);
}
// convert wstring to UTF-8 string
std::string wstring_to_utf8 (const std::wstring& str)
{
std::wstring_convert<std::codecvt_utf8<wchar_t>> myconv;
return myconv.to_bytes(str);
}
// https://stackoverflow.com/questions/1660492/utf-8-output-on-windows-console
// mike.dld's answer
class ConsoleStreamBufWin32 : public std::streambuf
{
public:
ConsoleStreamBufWin32(DWORD handleId, bool isInput);
protected:
// std::basic_streambuf
virtual std::streambuf* setbuf(char_type* s, std::streamsize n);
virtual int sync();
virtual int_type underflow();
virtual int_type overflow(int_type c = traits_type::eof());
private:
HANDLE const m_handle;
bool const m_isInput;
std::string m_buffer;
};
ConsoleStreamBufWin32::ConsoleStreamBufWin32(DWORD handleId, bool isInput) :
m_handle(::GetStdHandle(handleId)),
m_isInput(isInput),
m_buffer()
{
if (m_isInput)
{
setg(0, 0, 0);
}
}
std::streambuf* ConsoleStreamBufWin32::setbuf(char_type* /*s*/, std::streamsize /*n*/)
{
return 0;
}
int ConsoleStreamBufWin32::sync()
{
if (m_isInput)
{
::FlushConsoleInputBuffer(m_handle);
setg(0, 0, 0);
}
else
{
if (m_buffer.empty())
{
return 0;
}
std::wstring const wideBuffer = utf8_to_wstring(m_buffer);
DWORD writtenSize;
::WriteConsoleW(m_handle, wideBuffer.c_str(), wideBuffer.size(), &writtenSize, NULL);
}
m_buffer.clear();
return 0;
}
ConsoleStreamBufWin32::int_type ConsoleStreamBufWin32::underflow()
{
if (!m_isInput)
{
return traits_type::eof();
}
if (gptr() >= egptr())
{
wchar_t wideBuffer[128];
DWORD readSize;
if (!::ReadConsoleW(m_handle, wideBuffer, ARRAYSIZE(wideBuffer) - 1, &readSize, NULL))
{
return traits_type::eof();
}
wideBuffer[readSize] = L'\0';
m_buffer = wstring_to_utf8(wideBuffer);
setg(&m_buffer[0], &m_buffer[0], &m_buffer[0] + m_buffer.size());
if (gptr() >= egptr())
{
return traits_type::eof();
}
}
return sgetc();
}
ConsoleStreamBufWin32::int_type ConsoleStreamBufWin32::overflow(int_type c)
{
if (m_isInput)
{
return traits_type::eof();
}
m_buffer += traits_type::to_char_type(c);
return traits_type::not_eof(c);
}
template<typename StreamT>
inline void FixStdStream(DWORD handleId, bool isInput, StreamT& stream)
{
if (::GetFileType(::GetStdHandle(handleId)) == FILE_TYPE_CHAR)
{
stream.rdbuf(new ConsoleStreamBufWin32(handleId, isInput));
}
}
// some code are from this blog
// https://blog.csdn.net/witton/article/details/108087135
#define printf(fmt, ...) __fprint(stdout, fmt, ##__VA_ARGS__ )
int __vfprint(FILE *fp, const char *fmt, va_list va)
{
// https://stackoverflow.com/questions/7315936/which-of-sprintf-snprintf-is-more-secure
size_t nbytes = snprintf(NULL, 0, fmt, va) + 1; /* +1 for the '\0' */
char *str = (char*)malloc(nbytes);
snprintf(str, nbytes, fmt, va);
std::cout << str;
free(str);
return nbytes;
}
int __fprint(FILE *fp, const char *fmt, ...)
{
va_list va;
va_start(va, fmt);
int n = __vfprint(fp, fmt, va);
va_end(va);
return n;
}
int main()
{
FixStdStream(STD_INPUT_HANDLE, true, std::cin);
FixStdStream(STD_OUTPUT_HANDLE, false, std::cout);
FixStdStream(STD_ERROR_HANDLE, false, std::cerr);
// ...
std::cout << "\xc3\xbc" << std::endl;
printf("\xc3\xbc");
// ...
return 0;
}
The source code is saved in UTF-8 format, and build under Msys2's GCC and run under Windows 7 64bit. Here is the result
ü
ü