I am trying to wrap up a code that read gz files into a function, source code is taken from https://techoverflow.net/2013/11/03/c-iterating-lines-in-a-gz-file-using-boostiostreams/
My try
boost::iostreams::filtering_streambuf<boost::iostreams::input> func(std::string filename);
boost::iostreams::filtering_streambuf<boost::iostreams::input> func(std::string filename)
{
std::ifstream file(filename, std::ios_base::in | std::ios_base::binary);
boost::iostreams::filtering_streambuf<boost::iostreams::input> inbuf;
inbuf.push(boost::iostreams::gzip_decompressor());
inbuf.push(file);
return inbuf;
}
void mymainfunc(std::string filename)
{
//Convert streambuf to istream
std::istream ifstrm( func( filename));
std::string line;
while(std::getline(ifstrm, line)) {
std::cout << line << std::endl;
}
}
Code runs fine if not run through the function, i am doing something wrong in the return type I think.
Error: https://pastebin.com/kFpjYG0M
Streams are not copyable. In fact, this filteringstreambuf is not even movable.
So in this case, you will want to dynamically allocate and return by smart-pointer. However, even just returning the filtering streambuf will not work, because it would hold a reference to the ifstream. And that's a local.
So, maybe you need to package it up:
Live On Coliru
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <fstream>
namespace bio = boost::iostreams;
struct MySource {
using fisb = bio::filtering_istreambuf;
struct State {
State(std::string filename) : ifs(filename, std::ios::binary) {
buf.push(bio::gzip_decompressor());
buf.push(ifs);
}
fisb buf;
std::ifstream ifs;
std::istream is { &buf };
};
std::unique_ptr<State> _state;
operator std::istream&() const { return _state->is; }
};
MySource func(std::string filename) {
auto inbuf = std::make_unique<MySource::State>(filename);
return {std::move(inbuf)};
}
#include <iostream>
void mymainfunc(std::string filename)
{
auto source = func(filename);
std::istream& is = source;
std::string line;
while(std::getline(is, line)) {
std::cout << line << std::endl;
}
}
int main(){
mymainfunc("test.cpp.gz");
}
Alternative #1
You can simplify that:
Live On Coliru
struct MySource {
struct State {
State(std::string filename) : ifs(filename, std::ios::binary) {
is.push(bio::gzip_decompressor());
is.push(ifs);
}
std::ifstream ifs;
bio::filtering_istream is;
};
std::unique_ptr<State> _state;
operator std::istream&() const { return _state->is; }
};
Not dealing with the streambuffer separately makes it simpler.
Alternative #2
Not copying the whole thing has its own elegance:
Live On Coliru
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <fstream>
#include <iostream>
void mymainfunc(std::istream& is) {
std::string line;
while(std::getline(is, line)) {
std::cout << line << std::endl;
}
}
namespace bio = boost::iostreams;
int main(){
std::ifstream ifs("test.cpp.gz", std::ios::binary);
bio::filtering_istream is;
is.push(bio::gzip_decompressor());
is.push(ifs);
mymainfunc(is);
}
Related
I'm using boost gzip example code here.
I am attempting to compress a simple string test and am expecting the compressed string H4sIAAAAAAAACitJLS4BAAx+f9gEAAAA as shown in this online compressor
static std::string compress(const std::string& data)
{
namespace bio = boost::iostreams;
std::stringstream compressed;
std::stringstream origin(data);
bio::filtering_streambuf<bio::input> out;
out.push(bio::gzip_compressor(bio::gzip_params(bio::gzip::best_compression)));
out.push(origin);
bio::copy(out, compressed);
return compressed.str();
}
int main(int argc, char* argv[]){
std::cout << compress("text") << std::endl;
// prints out garabage
return 0;
}
However when I print out the result of the conversion I get garbage values like +I-. ~
I know that it's a valid conversion because the decompression value returns the correct string. However I need the format of the string to be human readable i.e. H4sIAAAAAAAACitJLS4BAAx+f9gEAAAA.
How can I modify the code to output human readable text?
Thanks
Motivation
The garbage format is not compatible with my JSON library where I will send the compressed text through.
The example site completely fails to mention they also base64 encode the result:
base64 -d <<< 'H4sIAAAAAAAACitJLS4BAAx+f9gEAAAA' | gunzip -
Prints:
test
In short, you need to also do that:
Live On Coliru
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <iostream>
#include <sstream>
#include <boost/archive/iterators/binary_from_base64.hpp>
#include <boost/archive/iterators/base64_from_binary.hpp>
#include <boost/archive/iterators/transform_width.hpp>
std::string decode64(std::string const& val)
{
using namespace boost::archive::iterators;
return {
transform_width<binary_from_base64<std::string::const_iterator>, 8, 6>{
std::begin(val)},
{std::end(val)},
};
}
std::string encode64(std::string const& val)
{
using namespace boost::archive::iterators;
std::string r{
base64_from_binary<transform_width<std::string::const_iterator, 6, 8>>{
std::begin(val)},
{std::end(val)},
};
return r.append((3 - val.size() % 3) % 3, '=');
}
static std::string compress(const std::string& data)
{
namespace bio = boost::iostreams;
std::istringstream origin(data);
bio::filtering_istreambuf in;
in.push(
bio::gzip_compressor(bio::gzip_params(bio::gzip::best_compression)));
in.push(origin);
std::ostringstream compressed;
bio::copy(in, compressed);
return compressed.str();
}
static std::string decompress(const std::string& data)
{
namespace bio = boost::iostreams;
std::istringstream compressed(data);
bio::filtering_istreambuf in;
in.push(bio::gzip_decompressor());
in.push(compressed);
std::ostringstream origin;
bio::copy(in, origin);
return origin.str();
}
int main() {
auto msg = encode64(compress("test"));
std::cout << msg << std::endl;
std::cout << decompress(decode64(msg)) << std::endl;
}
Prints
H4sIAAAAAAAC/ytJLS4BAAx+f9gEAAAA
test
I would like to parse either an stdin stream or a file. So I want a function/method to accept either of this.
Note that my goal is not to call read twice!
As istream is the base class for cin and ifstream` I should be able write this:
#include <iostream>
#include <fstream>
void read(std::istream &fp) {
while(!fp.eof()) {
std::string line;
std::getline(fp, line);
std::cout << line << std::endl;;
}
}
int main(int argc, char *argv[])
{
std::ifstream fp;
if (argc >= 2) {
fp.open(argv[1]);
if (!fp) abort();
}
else {
fp = std::cin;
}
read(fp);
if (fp.is_open()) {
fp.close();
}
return 0;
}
In C I can do the following with calling it with either read_content(stdin) or read_content(fp):
void read_content(FILE *file)
What is the proper way to do this in C++?
std::cin is an instance of std::istream and not derived from std::ifstream but the opposite is true.
Inheritance graph of Stream-based I/O:
(Taken from cppreference.com - Input/output library)
So, OPs intention can be performed using a reference or pointer to std::istream.
Demo:
#include <iostream>
#include <fstream>
void read(std::istream &fp) {
while(!fp.eof()) {
std::string line;
std::getline(fp, line);
std::cout << line << std::endl;;
}
}
int main(int argc, char *argv[])
{
std::ifstream fp;
std::istream &in = (argc >= 2)
? [&]() -> std::istream& {
fp.open(argv[1]);
if (!fp) abort();
return fp;
}()
: std::cin;
read(in);
if (fp.is_open()) {
fp.close();
}
return 0;
}
Compiled on coliru
Note:
while (!fp.eof()) {
should be replaced by
while (fp) {
The reason for this has been thoroughly discussed in
SO: Why is iostream::eof inside a loop condition (i.e. while (!stream.eof())) considered wrong?.
In C I can do the following with calling it with either read_content(stdin) or read_content(fp):
Yes, and in C++ you should just call either read(std::cin) or read(fp). It's exactly the same.
The line
fp = std::cin;
is the wrong thing to do, as std::cin is only declared as a std::istream. There is no std::ifstream constructor taking an istream, you don't want an independent object here anyway, and if std::cin is really an object of some type derived from std::ifstream, you'd slice it.
I want to write a simple program that depending on the options passed it the executable will print the output to the screen or to a file. The program is simple.
#include<iostream>
int main(int argc, char* argv[]){
... process options...
std::ostream& out = ... // maybe std::cout, maybe a *new* std::ofstream;
out << "content\n";
}
Is there a good idiom to make out refer alternatively to std::cout or a file stream at runtime?
I tried with pointers, but it is horrible. I couldn't avoid using pointers (Not to mention that more ugly code is needed to delete the pointer later).
#include<iostream>
#include<ofstream>
int main(int argc, char* argv[]){
std::string file = argc>1?argv[1]:"";
std::clog << "file: " << file << '\n';
// if there is no argument it will print to screen
std::ostream* out = (file=="")?&std::cout:(new std::ofstream(file)); // horrible code
*out << "content" << std::endl;
if(out != &std::cout) delete out;
}
I don't know, perhaps there is some feature of C++ streams that allows this. Perhaps I have to use some kind of type erasure. The problem, I think, is that std::cout is something that already exists (is global), but std::ofstream is something that has to be created.
I managed to use open and avoid pointers but it is still ugly:
int main(int argc, char* argv[]){
std::string file = argc>1?argv[1]:"";
std::clog << "file: " << file << '\n';
std::ofstream ofs;
if(file != "") ofs.open(file);
std::ostream& out = (file=="")?std::cout:ofs;
out << "content" << std::endl;
}
My preference is to use streams with suitable stream buffers installed. Here is one way direct output to a file or to std::cout:
#include <iostream>
#include <fstream>
int main(int ac, char* av) {
std::ofstream ofs;
if (1 < ac) {
ofs.open(av[1]);
// handle errors opening the file here
}
std::ostream os(file? file.rdbuf(): std::cout.rdbuf());
// use os ...
}
So much over-engineering.
#include <iostream>
#include <fstream>
int main(int argc, char* argv[]) {
std::ofstream ofs(argc > 1 ? argv[1] : "");
std::ostream& os = ofs.is_open() ? ofs : std::cout;
// use os ...
}
A runtime binding of the desired stream will pretty much need to look like what you already have.
On the pointer issue, sure you can clean it up a bit... maybe something like this? This is assuming you only want to create the ofstream if the argument exists.
int main(int argc, char* argv[]){
std::string file = argc > 1 ? argv[1] : "";
std::clog << "file: " << file << '\n';
// if there is no argument it will print to screen
std::unique_ptr<std::ostream> fp;
if (file == "")
fp = std::make_unique<std::ofstream>(file);
std::ostream& out = (fp && fp->is_open()) ? std::cout : *fp; // not so horrible code
out << "content" << std::endl;
}
If the dynamic object is not required, the easiest may be something list this;
int main(int argc, char* argv[]){
std::string filename = (argc > 1) ? argv[1] : "";
std::ofstream file(filename);
// if there is no argument (file) it will print to screen
std::ostream& out = file.is_open() ? file : std::cout;
out << "content" << std::endl;
}
I often use something like this for command-line tools:
int main(int, char* argv[])
{
std::string filename;
// args processing ... set filename from command line if present
if(argv[1])
filename = argv[1];
std::ofstream ofs;
// if a filename was given try to open
if(!filename.empty())
ofs.open(filename);
// bad ofs means tried to open but failed
if(!ofs)
{
std::cerr << "Error opeing file: " << filename << '\n';
return EXIT_FAILURE;
}
// Here either ofs is open or a filename was not provided (use std::cout)
std::ostream& os = ofs.is_open() ? ofs : std::cout;
// write to output
os << "Some stuff" << '\n';
return EXIT_SUCCESS;
}
You could use a shared pointer to a stream for the polymorphic behavior:
#include <memory>
#include <fstream>
#include <sstream>
#include <iostream>
void nodelete(void*) {}
std::shared_ptr<std::ostream> out_screen_stream() { return std::shared_ptr<std::ostream>(&std::cout, nodelete); }
std::shared_ptr<std::ostream> out_file_stream() { return std::make_shared<std::ofstream>(); }
std::shared_ptr<std::ostream> out_string_stream() { return std::make_shared<std::ostringstream>(); }
int main ()
{
std::shared_ptr<std::ostream> out;
// case condition:
out = out_screen_stream();
out = out_file_stream();
out = out_string_stream();
*out << "content" << std::endl;
return 0;
}
Note: A std::shared_ptr allows managing different possible streams, where some streams should not get deleted (e.g.: std::cout).
Similar, but with std::unique_ptr:
#include <memory>
#include <fstream>
#include <sstream>
#include <iostream>
class Deleter
{
public:
Deleter(bool use_delete = true) : use_delete(use_delete) {}
template <typename T>
void operator () (const T* p) {
if(use_delete)
delete p;
}
bool nodelete() const { return ! use_delete; }
private:
bool use_delete;
};
using unique_ostream_ptr = std::unique_ptr<std::ostream, Deleter>;
unique_ostream_ptr out_screen_stream() { return unique_ostream_ptr(&std::cout, false); }
unique_ostream_ptr out_file_stream() { return unique_ostream_ptr{ new std::ofstream }; }
unique_ostream_ptr out_string_stream() { return unique_ostream_ptr{ new std::ostringstream }; }
int main ()
{
unique_ostream_ptr out;
// case condition:
out = out_screen_stream();
out = out_file_stream();
out = out_string_stream();
*out << "content" << std::endl;
return 0;
}
Maybe a reference?
#include<iostream>
#include<ofstream>
int main(int argc, char* argv[])
{
auto &out = std::cout;
std::ofstream outFile;
std::string fileName = argc>1?argv[1]:"";
std::clog << "file: " << file << '\n';
// if there is no argument it will print to screen
if(!fileName.empty())
{
outFile.open(fileName);
out = outFile;
}
out<<"one, one, two";
return 0;
}
I got this code form the boost library.
http://www.boost.org/doc/libs/1_42_0/doc/html/boost_propertytree/tutorial.html
This is the xml file they have
<debug>
<filename>debug.log</filename>
<modules>
<module>Finance</module>
<module>Admin</module>
<module>HR</module>
</modules>
<level>2</level>
</debug>
The code to load these values and print them is
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>
#include <boost/foreach.hpp>
#include <string>
#include <set>
#include <exception>
#include <iostream>
struct debug_settings
{
std::string m_file; // log filename
int m_level; // debug level
std::set<std::string> m_modules; // modules where logging is enabled
void load(const std::string &filename);
void save(const std::string &filename);
};
void debug_settings::load(const std::string &filename)
{
using boost::property_tree::ptree;
ptree pt;
read_xml(filename, pt);
m_file = pt.get<std::string>("debug.filename");
m_level = pt.get("debug.level", 0);
BOOST_FOREACH(ptree::value_type &v, pt.get_child("debug.modules"))
m_modules.insert(v.second.data());
}
void debug_settings::save(const std::string &filename)
{
using boost::property_tree::ptree;
ptree pt;
pt.put("debug.filename", m_file);
pt.put("debug.level", m_level);
BOOST_FOREACH(const std::string &name, m_modules)
pt.put("debug.modules.module", name,true);
write_xml(filename, pt);
}
int main()
{
try
{
debug_settings ds;
ds.load("debug_settings.xml");
ds.save("debug_settings_out.xml");
std::cout << "Success\n";
}
catch (std::exception &e)
{
std::cout << "Error: " << e.what() << "\n";
}
return 0;
}
But it gives me an error
/usr/include/boost/property_tree/detail/ptree_implementation.hpp:769: error: request for member ‘put_value’ in ‘tr’, which is of non-class type ‘bool’
Can anyone tell me what I am missing?
Seems they have replaced the put () function ... So if I changed the line
"pt.put("debug.modules.module", name,true);"
to
"pt.add("debug.modules.module", name);"
it works fine. Thank you.
I saw a useful start here:
http://www.cs.technion.ac.il/~imaman/programs/teestream.html
And it works great to make a new stream which goes to both clog and a log file.
However, if I try to redefine clog to be the new stream it does not work because the new stream has the same rdbuf() as clog so the following has no effect:
clog.rdbuf(myTee.rdbuf());
So how can I modify the tee class to have its own rdbuf() which can then be the target of clog?
Thanks.
-William
If you really want to keep using std::clog for the tee instead of sending output to a different stream, you need to work one level lower: Instead of deriving from ostream, derive from streambuf. Then you can do this:
fstream logFile(...);
TeeBuf tbuf(logFile.rdbuf(), clog.rdbuf());
clog.rdbuf(&tbuf);
For more information on how to derive your own streambuf class, see here.
You don't want to do what your've trying to do because the 'tee' is not working at the rdbuf level. So setting the rdbuf to something else will not work, the output will only go to one stream.
You need to follow there example:
e.g.
fstream clog_file(...);
xstream clog_x(...);
TeeStream clog(clog_file, clog_x);
then use clog everywhere instead of your original clog.
Here is the class I created that seems to do the job, thanks to all who helped out!
-William
class TeeStream : public std::basic_filebuf<char, std::char_traits<char> >
{
private:
class FileStream : public std::ofstream {
public:
FileStream()
: logFileName("/my/log/file/location.log") {
open(logFileName.c_str(), ios::out | ios::trunc);
if (fail()) {
cerr << "Error: failed to open log file: " << logFileName << endl;
exit(1);
}
}
~FileStream() {
close();
}
const char *getLogFileName() const {
return logFileName.c_str();
}
private:
const string logFileName;
};
public:
typedef std::char_traits<char> traits;
typedef std::basic_filebuf<char, traits> baseClass;
TeeStream()
: baseClass(),
_logOutputStream(),
_clogBuf(clog.rdbuf()),
_fileBuf(_logOutputStream.rdbuf()) {
clog.rdbuf(this);
_logOutputStream << "Log file starts here:" << endl;
}
~TeeStream() {
clog.rdbuf(_clogBuf);
}
int_type overflow(char_type additionalChar =traits::eof()) {
const int_type eof = traits::eof();
const char_type additionalCharacter = traits::to_char_type(additionalChar);
const int_type result1 = _clogBuf->sputc(additionalCharacter);
const int_type result2 = _fileBuf->sputc(additionalCharacter);
if (traits::eq_int_type(eof, result1)) {
return eof;
} else {
return result2;
}
}
int sync() {
const int result1 = _clogBuf->pubsync();
const int result2 = _fileBuf->pubsync();
if (result1 == -1) {
return -1;
} else {
return result2;
}
}
private:
FileStream _logOutputStream;
streambuf * const _clogBuf;
streambuf * const _fileBuf;
};
I would just use the Boost iostreams stuff to do it.
#include <iostream>
#include <fstream>
#include <boost/iostreams/tee.hpp>
#include <boost/iostreams/stream.hpp>
int main(const int a_argc, const char *a_args[])
{
namespace io = boost::iostreams;
typedef io::tee_device<std::ofstream, std::ostream> TeeDevice;
typedef io::stream<TeeDevice> TeeStream;
std::ofstream flog("logFile.txt");
//We need to copy clog, otherwise we get infinite recursion
//later on when we reassign clog's rdbuf.
std::ostream clogCopy(std::clog.rdbuf());
TeeDevice logTee(flog, clogCopy);
TeeStream logTeeStream(logTee);
logTeeStream << "This text gets clogged and flogged." << std::endl;
//Modify clog to automatically go through the tee.
std::streambuf *originalRdBuf = std::clog.rdbuf(logTeeStream.rdbuf());
std::clog << "This text doesn't only get clogged, it's flogged too." << std::endl;
std::clog.rdbuf(originalRdBuf);
std::clog << "This text avoids flogging." << std::endl;
}