How to find the index in Poco::DeflatingOutputStream - c++

Hi I am new to usage of Poco , can you please help me to find a way to get the index/position during the writing into deflating stream so that I can truncate the invalid data and make sure my file contains only valid data.
#include <stdexcept>
#include <stdarg.h>
#include <map>
#include <iostream>
#include <cstring>
#include <fstream>
#include <Poco/DeflatingStream.h>
#include <stdio.h>
#include <limits>
#include <stdio.h>
#include <unistd.h>
using namespace std;
std::ofstream* ostr;
Poco::DeflatingOutputStream* ofstr;
string fileName="/home/lamb/Cpp/simple.gzip";
bool written = false;
// int lastsucessfulwrite;
compress(){
*ofstr << "\t<xyz>\n";
*ofstr << "\t</xyz>\n";
*ofstr << " who=\"";
*ofstr << "/>\n";
written = true;
/* "lastsucessfulwrite" How to store the index of ofstr , in case of normal files we use ftell but in DeflatingOutputStream how to get index so that I can erase it later based on this value */
}
timer(){
sleep(2);
// 2 second
written = false ;
}
close(){
ofstr->close();
delete ofstr;
ofstr = NULL;
ostr->close();
delete ostr;
ostr = NULL;
}
int main(){
ostr = new std::ofstream;
ostr->exceptions(std::ofstream::failbit|std::ofstream::badbit);
ostr->open(_fileName.c_str(), std::ios::binary | std::ios::app);
ofstr = new Poco::DeflatingOutputStream(*_ostr,
Poco::DeflatingStreamBuf::STREAM_GZIP);
ofstr->precision(std::numeric_limits<double>::digits10);
string data1 = "hello';
string data2 = "hello';
string data3 = "hello';
written = false ;
timer()//start
compress(data1);
if(written)
{
compress(data2);
}
if(written)
{
compress(data2);
}
if(written)
{
compress(data3);// timeup and time() is inovked and part of compress() is executed
}
// Now I would like to use lastsucessfulwrite as the key and truncate the paritally witten data3
// In case of normal file we use "truncate" system call
close();
}

You can use any standard C++ stream functions with Poco streams.
streampos pos = ofstr->tellp()

Related

How to send a file by a zmq multipart message?

I'm trying to send a video using a ZeroMQ infrastructure and I split the video into chunks to send it. When I do it and put the video into a vector to send it through zmq::send_multipart I get a very high usage of RAM memory and so times later I get a segmentation fault error.
My headache is that when I comment on the line that sends the multipart message and runs the program, the vector is made normally and I don't get the segmentation error, although the consumption of RAM memory is not so heavy.
Can someone give me a tip about how to send this file?
Server code:
#include <fstream>
#include <sstream>
#include <chrono>
#include <thread>
#include <iostream>
#include <future>
#include <zmq.hpp>
#include <zmq_addon.hpp>
using namespace std::chrono_literals;
const int size1MB = 1024 * 1024;
template <typename T>
void debug(T x)
{
std::cout << x << std::endl;
}
//Generate new chunks
std::unique_ptr<std::ofstream> createChunkFile(std::vector<std::string> &vecFilenames)
{
std::stringstream filename;
filename << "chunk" << vecFilenames.size() << ".mp4";
vecFilenames.push_back(filename.str());
return std::make_unique<std::ofstream>(filename.str(), std::ios::trunc);
}
//Split the file into chunks
void split(std::istream &inStream, int nMegaBytesPerChunk, std::vector<std::string> &vecFilenames)
{
std::unique_ptr<char[]> buffer(new char[size1MB]);
int nCurrentMegaBytes = 0;
std::unique_ptr<std::ostream> pOutStream = createChunkFile(vecFilenames);
while (!inStream.eof())
{
inStream.read(buffer.get(), size1MB);
pOutStream->write(buffer.get(), inStream.gcount());
++nCurrentMegaBytes;
if (nCurrentMegaBytes >= nMegaBytesPerChunk)
{
pOutStream = createChunkFile(vecFilenames);
nCurrentMegaBytes = 0;
}
}
}
int main()
{
zmq::context_t context(1);
zmq::socket_t socket(context, zmq::socket_type::rep);
socket.bind("tcp://*:5555");
std::ifstream img("video2.mp4", std::ifstream::in | std::ios::binary);
std::ifstream aux;
std::vector<std::string> vecFilenames;
std::vector<zmq::const_buffer> data;
std::ostringstream os;
std::async(std::launch::async, [&img, &vecFilenames]() {
split(img, 100, vecFilenames);
});
img.close();
zmq::message_t message, aux2;
socket.recv(message, zmq::recv_flags::none);
//Put the chunks into the vector
std::async([&data, &aux, &os, &vecFilenames]() {
for (int i = 0; i < vecFilenames.size(); i++)
{
std::async([&aux, &i]() {
aux.open("chunk" + std::to_string(i) + ".mp4", std::ifstream::in | std::ios::binary);
});
os << aux.rdbuf();
data.push_back(zmq::buffer(os.str()));
os.clear();
aux.close();
}
});
//Send the vector for the client
std::async([&socket, &data] {
zmq::send_multipart(socket, data);
});
}
Client-side:
#include <fstream>
#include <sstream>
#include <iostream>
#include <thread>
#include <chrono>
#include <string>
#include <zmq.hpp>
#include <zmq_addon.hpp>
#include <queue>
#include <deque>
#include <future>
#include <vector>
using namespace std::chrono_literals;
template <typename T>
void debug(T x)
{
std::cout << x << std::endl;
}
int main()
{
zmq::context_t context(1);
zmq::socket_t socket(context, zmq::socket_type::req);
socket.connect("tcp://localhost:5555");
std::ofstream img("teste.mp4", std::ios::out | std::ios::binary);
socket.send(zmq::buffer("ok\n"), zmq::send_flags::none);
std::vector<zmq::message_t> send_msgs;
zmq::message_t size;
std::async([&send_msgs, &img, &socket] {
zmq::recv_multipart(socket, std::back_inserter(send_msgs));
while (send_msgs.size())
{
img << send_msgs[0].to_string();
send_msgs.erase(send_msgs.begin());
}
});
}
An attempt to move all data via multipart-message does collect all data into one immense BLOB, plus add duplicate O/S-level transport-class specific buffers and the most probable result is a crash.
Send individual blocks of the video-BLOB as individual simple-message payloads and reconstruct the BLOB ( best via indexed numbering, having an option to re-request any part, that did not arrive to the receiver-side ).
Using std::async mode with a REQ/REP seems to be a bit tricky for this Archetype must keep its dFSA interleaved sequence of .recv()-.send()-.recv()-.send()-...ad infinitum... as it falls into an unsalvagable mutual deadlock if failed to do so.
For streaming video (like for CV / scene-detection), there are more tricks to put in - one of which is to use ZMQ_CONFLATE option, so as to send but the very recent video-frame, not losing time on "archaic" scene-images, that are already part of the history, and thus delivering always but the very recent video-frame to the receiving-side processing.

Ambiguous pointer; <SiHCollection> is ambiguous

below I've posted my code which is meant to store hits collections in HCE (hit collection of events).
The code compiles successfully but on running the program, the following error is printed to the terminal seven times:
< SiHCollection> is ambiguous.
I have a feeling it is because I am using namespace std although I don't know how to amend the code. Any thoughts?
#include "SiSD.h"
#include "SiHit.h"
#include "G4HCofThisEvent.hh"
#include "G4Step.hh"
#include "G4ThreeVector.hh"
#include "G4SDManager.hh"
#include "G4ios.hh"
#include "G4UnitsTable.hh"
#include <fstream>
#include <iostream>
#include <sstream>
using namespace std;
extern ofstream outfile;
SiSD::SiSD(G4String name)
:G4VSensitiveDetector(name)
{
collectionName.insert("SiHCollection");
}
SiSD::~SiSD(){ }
void SiSD::Initialize(G4HCofThisEvent* HCE)
{
SiHCollection = new SiHitsCollection(SensitiveDetectorName,
collectionName[0]);
static G4int HCID = -1;
if(HCID<0)
{
HCID = G4SDManager::GetSDMpointer()->GetCollectionID(collectionName[0]);
}
HCE->AddHitsCollection(HCID, SiHCollection);
}
G4bool SiSD::ProcessHits(G4Step* aStep, G4TouchableHistory*)
{
if(aStep->GetTrack()->GetTrackID() > 0) {
G4double edep = aStep->GetTotalEnergyDeposit();
if(edep==0) return false;
SiHit* aHit = new SiHit();
aHit->SetEdep(edep);
SiHCollection->insert(aHit);
return true;
} else return false;
}

c++ put functions in separate source files

Currently I have an unique source file (*.cpp) where all my functions are working right. Now i'm trying to take some of them out into separate source files and including them into main source with no success.
My current project is as follows:
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <windows.h>
#define _SQLNCLI_ODBC_
#include <sqlext.h>
#include <sqlncli.h>
using namespace std;
using std::cout;
using std::ifstream;
/*This is one of the functions to be put in separate file:*/
string *ReadPageAsignations ( const char* RutayNombre, const char* Page )
{
bool MisionCumplida = false;
bool EncabezadoListo = false;
int i = 0;
int j = 0;
char * pch;
char istr[256];
const int NUM_DATA = 15;
static string data[NUM_DATA];
std::stringstream InputString;
ifstream inputFile(RutayNombre);
if (inputFile.is_open())
{
while (inputFile.good() && MisionCumplida == false)
{
i = 0;
inputFile.getline(istr,256);
pch = strtok (istr,":");
if (string(pch) == "[Pagina]")
{
EncabezadoListo = true;
}
else
{
EncabezadoListo = false;
}
if (string(pch) == Page)
{
MisionCumplida = true;
}
while (pch != NULL)
{
if ((EncabezadoListo == true) || (MisionCumplida == true))
{
data[i] = data[i] + " " + string(pch);
}
pch = strtok (NULL, ",");
i++;
}
}
inputFile.close();
return data;
}
} //End of function 'ReadPageAsignations'
/*This is another function where my function "ReadPageAsignations' get called -- btw, I want also this function to be in a separate source file.*/
void DeliverHtml (const char* page){//const char* RutayNombre ) {
string *p;
char * pch;
size_t pos;
string RutayNombre;
RutayNombre = "../Substructure/Templates/" + SearchConfigValue( "../Substructure/Conf/Config-Templates.txt", "htmlTemplate:");
const char *RutayNombreConfigCompos = "../Substructure/Conf/Config-Composition.txt";
string RutayNombreParaInsertar;
string token, token1, token2;
string line, lineRead, lineToInsert;
char * StrToTokenize2;
string StrToTokenize1;
p=ReadPageAsignations( RutayNombreConfigCompos, page); //Here, I call the function I want in a separate file
...
}
/*And here is the main() function*/
int main()
{
char *value = "page=Home";
if (NULL!=strstr(getenv("QUERY_STRING"), "page="))
{
value = getenv("QUERY_STRING");
}
char *posCh = strstr(value, "=");
DeliverHtml(&posCh[0]+1);
return 0;
}
For the first function, I have tried creating the header file 'ReadPageAsignations.h' and a source file 'ReadPageAsignations.cpp'.
Header file 'ReadPageAsignations.h' containing:
#ifndef READPAGEASIGNATIONS_H_INCLUDED
#define READPAGEASIGNATIONS_H_INCLUDED
string *ReadPageAsignations ( const char* RutayNombre, const char* Page );
#endif // READPAGEASSIGNATIONS_H_INCLUDED
Source file 'ReadPageAsignations.cpp' for separate function containing:
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
using namespace std;
using std::cout;
using std::ifstream;
string *ReadPageAsignations ( const char* RutayNombre, const char* Page )
{
bool MisionCumplida = false;
bool EncabezadoListo = false;
int i = 0;
int j = 0;
char * pch;
char istr[256];
const int NUM_DATA = 15; /*El numero de elementos debe coincidir con el iterador en la función Deliverhtml.*/
static string data[NUM_DATA];
std::stringstream InputString;
ifstream inputFile(RutayNombre); //Abre el archivo y lo asigna al stream inputFile.
if (inputFile.is_open()) //Chequea que el archivo esté abierto.
{
while (inputFile.good() && MisionCumplida == false)
{
i = 0;
inputFile.getline(istr,256);
pch = strtok (istr,":");
if (string(pch) == "[Pagina]")
{
EncabezadoListo = true;
}
else
{
EncabezadoListo = false;
}
if (string(pch) == Page)
{
MisionCumplida = true;
}
while (pch != NULL)
{
if ((EncabezadoListo == true) || (MisionCumplida == true))
{
data[i] = data[i] + " " + string(pch);
}
pch = strtok (NULL, ",");
i++;
}
}
inputFile.close();
return data;
}
} //End function
and, main project containing:
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <windows.h>
#define _SQLNCLI_ODBC_
#include <sqlext.h>
#include <sqlncli.h>
#include "ReadPageAsignations.h" //Here I #include the function definition file (header)
using namespace std;
using std::cout;
using std::ifstream;
...
}
I've got a lot of compiling errors:
\ReadPageAsignations.h|4|error C2143: syntax error : missing ';' before '*'|
\ReadPageAsignations.h|4|error C4430: missing type specifier - int assumed. Note: C++ does not support default-int|
\ReadPageAsignations.h|4|error C4430: missing type specifier - int assumed. Note: C++ does not support default-int|
main.cpp|20|error C2872: 'string' : ambiguous symbol|
...
I'm working Code::blocks 13.12 with MS Visual C++ 2005/2008 compiler.
any help will be highly appreciated, thanks in advance.
The error is telling you that when it tried to parse the header file it encountered the symbol string and doesn't recognize it. Adding #include <string> to your header file and fully qualifying the string type as std::string should correct the problem.
You should put #include <string> in your header file and remove it from your .cpp file
as following:
main.cpp
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <windows.h>
#define _SQLNCLI_ODBC_
#include <sqlext.h>
#include <sqlncli.h>
#include "ReadPageAsignations.h"
...
note: including header file with the same name of .cpp file , include both.
ReadPageAsignations.h
#ifndef READPAGEASIGNATIONS_H_INCLUDED
#define READPAGEASIGNATIONS_H_INCLUDED
#include <string> //<-----This line, include string header
std::string *ReadPageAsignations ( const char* RutayNombre, const char* Page );
#endif // READPAGEASSIGNATIONS_H_INCLUDED
ReadPageAsignations.cpp
#include <iostream>
#include <fstream>
#include <sstream>
#include "ReadPageAsignations.h" // <--- add the header file here
//#include <string> <---remove it already included in the header file
using namespace std;
//using std::cout; <--remove this you already used namespace std
//using std::ifstream; <--remove this you already used namespace std
string *ReadPageAsignations ( const char* RutayNombre, const char* Page )
{
... } //End function

XDR serializing variable lenght array of string

I am serializing a packet over XDR but i do not understand how to provide vector of string. I have here a small fully working serialization / deserialization for a std::vector of uint64_t. Here my code:
Serializer:
#include <stdio.h>
#include <iostream>
#include <rpc/rpc.h>
#include <vector>
#define MAX_LENGTH_ 100
int main(void)
{
XDR xdr;
xdrstdio_create(&xdr, stdout, XDR_ENCODE);
std::vector<uint64_t> ids; // vector i want to send
ids.push_back(1);
ids.push_back(2);
ids.push_back(3);
// serializing the vector
uint64_t *_ids = &ids[0];
uint32_t size = ids.size();
xdr_array(&xdr,(char**)(&_ids), &size, MAX_LENGTH_,sizeof(uint64_t),(xdrproc_t)xdr_u_long);
return 1;
}
Deserializer:
#include <stdio.h>
#include <iostream>
#include <rpc/rpc.h>
#include <vector>
#define MAX_LENGTH_ 100
int main(void)
{
XDR xdrs;
xdrstdio_create(&xdrs, stdin, XDR_DECODE);
uint64_t *ids_ = new uint64_t[MAX_LENGTH_];
uint32_t size;
bool status = xdr_array(&xdrs,(char**)(&ids_), &size, MAX_LENGTH_,
sizeof(uint64_t), (xdrproc_t)xdr_u_long);
std::vector<uint64_t> ids(ids_,ids_+size);
for(std::vector<uint64_t>::iterator it = ids.begin(); it != ids.end(); ++it)
{
std::cout << *it <<std::endl;
}
return 1;
}
The following code works... running ./serializer | ./deserializer i obtain 1 2 3. Now I do not know how to handle having to serialize std::vector<std::string>. A single string works well using xdr_string.
http://linux.die.net/man/3/xdr_array
Any help would be very much appreciated!
EDIT:
I have tried the following:
Serializer:
#include <stdio.h>
#include <iostream>
#include <rpc/rpc.h>
#include <vector>
#include <algorithm>
#include <cstring>
#define MAX_VECTOR_LENGTH_ 100
#define MAX_STRING_LENGTH_ 50
char *convert(const std::string & s)
{
char *pc = new char[s.size()+1];
std::strcpy(pc, s.c_str());
return pc;
}
int main(void)
{
XDR xdr;
xdrstdio_create(&xdr, stdout, XDR_ENCODE);
std::vector<std::string> messages; // vector i want to send
messages.push_back("this is");
messages.push_back("my string");
messages.push_back("vector test");
// transform the vector to c style
std::vector<char*> messagesCStyle;
std::transform(messages.begin(), messages.end(), std::back_inserter(messagesCStyle), convert);
// serializing the vector
char **_messages = &messagesCStyle[0];
uint32_t size = messages.size();
xdr_array(&xdr,(char**)(&_messages), &size, MAX_VECTOR_LENGTH_ * MAX_STRING_LENGTH_,sizeof(char),(xdrproc_t)xdr_string);
return 1;
}
Deserializer:
#include <stdio.h>
#include <iostream>
#include <rpc/rpc.h>
#include <vector>
#define MAX_VECTOR_LENGTH_ 100
#define MAX_STRING_LENGTH_ 50
int main(void)
{
XDR xdrs;
xdrstdio_create(&xdrs, stdin, XDR_DECODE);
std::vector<char*> messagesCStyle_;
uint32_t size;
bool status = xdr_array(&xdrs,(char**)(&messagesCStyle_), &size, MAX_VECTOR_LENGTH_,
MAX_STRING_LENGTH_, (xdrproc_t)xdr_string);
for(std::vector<char*>::iterator it = messagesCStyle_.begin(); it != messagesCStyle_.end(); ++it)
{
std::cout << *it <<std::endl;
}
return 1;
}
I am pretty sure the code for the Serializer is not best but at least it seams to work. However the deserializer does not!! I think the problem is related to the fact that i do not know how much memory to allocate before calling the xdr_array. Any help?
I made it work:
Encoder:
#include <stdio.h>
#include <iostream>
#include <rpc/rpc.h>
#include <vector>
#include <algorithm>
#include <cstring>
#define MAX_VECTOR_LENGTH_ 100
#define MAX_STRING_LENGTH_ 50
char *convert(const std::string & s)
{
char *pc = new char[s.size()+1];
std::strcpy(pc, s.c_str());
return pc;
}
int main(void)
{
XDR xdr;
xdrstdio_create(&xdr, stdout, XDR_ENCODE);
std::vector<std::string> messages; // vector i want to send
messages.push_back("this is");
messages.push_back("my string");
messages.push_back("vector test");
messages.push_back("this is a relatively long string!!!");
// transform the vector to c style
std::vector<char*> messagesCStyle;
std::transform(messages.begin(), messages.end(),
std::back_inserter(messagesCStyle),
[](const std::string & s){
char *pc = new char[s.size()+1];
std::strcpy(pc, s.c_str());
return pc;
});
// serializing the vector
char **_messages = &messagesCStyle[0];
uint32_t size = messages.size();
xdr_array(&xdr,(char**)(&_messages), &size, MAX_VECTOR_LENGTH_ * MAX_STRING_LENGTH_,sizeof(char*),(xdrproc_t)xdr_string);
return 1;
}
Decoder:
#include <stdio.h>
#include <iostream>
#include <rpc/rpc.h>
#include <vector>
#define MAX_VECTOR_LENGTH_ 100
#define MAX_STRING_LENGTH_ 50
int main(void)
{
XDR xdrs;
uint32_t size;
char** buffer = NULL;
xdrstdio_create(&xdrs, stdin, XDR_DECODE);
bool status = xdr_array(&xdrs, (char**) &buffer, &size, MAX_VECTOR_LENGTH_,
sizeof(char*), (xdrproc_t)xdr_string);
std::cout << "status: " << status << std::endl;
std::cout << "size: " << size << std::endl;
std::vector<std::string> stringMessages_(buffer, buffer + size);
for(std::vector<std::string>::iterator it = stringMessages_.begin(); it != stringMessages_.end(); ++it)
{
std::cout << *it <<std::endl;
}
for (int i = 0; i < size; i++) {
free(buffer[i]);
}
free(buffer);
return 1;
}

howto: Read input and store it in another file

I want to make a program that reads the highest value from one file and stores it in another. I've read about ifstream and ofstream but how do I let the ofstream store the highest value from the instream in another file? Here is what I have so far:
#include <iostream>
#include <fstream>
#include <string>
#include <algorithm>
#include <iterator>
#include <vector>
using namespace std;
struct CsvWhitespace : ctype<char> {
static const mask* make_table() {
static vector<mask> v{classic_table(), classic_table() + table_size};
v[','] |= space; // comma will be classified as whitespace
return v.data();
}
CsvWhitespace(size_t refs = 0) : ctype{make_table(), false, refs} {}
} csvWhitespace;
int main() {
string line;
ifstream myfile ("C:/Users/Username/Desktop/log.csv");
ofstream myfile2 ("C:/Users/Username/Desktop/log2.csv");
return 0;
}
auto v = vector<int>{};
myfile.imbue(locale{myfile.getloc(), &csvWhitespace});
copy(istream_iterator<int>{myfile}, istream_iterator<int>{}, back_inserter(v));
myfile2 << *max_element(begin(v), end(v));
}
Thanks in advance :)
You could just copy from the one file in the other, without having to worry about the format, by treating them in binary mode. Here is an example:
#include <stdio.h>
#include <string.h>
#define bufSize 1024
int main(int argc, char *argv[])
{
FILE *ifp, *ofp;
char buf[bufSize];
if (argc != 3)
{
fprintf(stderr,
"Usage: %s <soure-file> <target-file>\n", argv[0]);
return 1;
}
if ((ifp = fopen(argv[1], "rb")) == NULL)
{ /* Open source file. */
perror("fopen source-file");
return 1;
}
if ((ofp = fopen(argv[2], "wb")) == NULL)
{ /* Open target file. */
perror("fopen target-file");
return 1;
}
while (fgets(buf, sizeof(buf), ifp) != NULL)
{ /* While we don't reach the end of source. */
/* Read characters from source file to fill buffer. */
/* Write characters read to target file. */
fwrite(buf, sizeof(char), strlen(buf), ofp);
}
fclose(ifp);
fclose(ofp);
return 0;
}
which was given as an example in IP, source. You just need to specify the cmd arguments as the desired files.
You can do it like this. Live example using cin and cout rather than files.
#include <iostream>
#include <fstream>
#include <string>
#include <algorithm>
#include <iterator>
#include <vector>
using namespace std;
struct CsvWhitespace : ctype<char> {
static const mask* make_table() {
static vector<mask> v{classic_table(), classic_table() + table_size};
v[','] |= space; // comma will be classified as whitespace
return v.data();
}
CsvWhitespace(size_t refs = 0) : ctype{make_table(), false, refs} {}
};
int main() {
string line;
ifstream myfile("log.csv");
ofstream myfile2("log2.csv");
auto v = vector<int>{};
myfile.imbue(locale{myfile.getloc(), new CsvWhitespace{}});
copy(istream_iterator<int>{myfile}, istream_iterator<int>{}, back_inserter(v));
myfile2 << *max_element(begin(v), end(v));
}