Remove carriage return from json text - c++

I get json text returned from an api call and I run it trough a script JSON Serialization and Deserialization from here https://www.mql5.com/en/code/13663.
My issue is that it only process the first few parts of the json because of what I believe to a linebreak/carriage inside the json structure. I dont get any error message, just an array containing everything before that line break.
I dont want to remove the line breaks inside the text fields in the json, only the returns inside the json structure. It is in the place each time just after {"ok":true,"result":[{"update_id":568022212,
Here is a full section
{"ok":true,"result":[{"update_id":568022212,
"channel_post":{"message_id":436,"chat":{"id":-1001436032340,"title":"FORTUNA","type":"channel"},"date":1588899840,"reply_to_message":{"message_id":372,"chat":{"id":-1001436032340,"title":"FORTUNA","type":"channel"},"date":1588838583,"text":"A\nGbpusd buy now 1.2360\nSl 1.2280 \nTp open\n","entities":[{"offset":52,"length":11,"type":"mention"}]},"text":"A\n42 pips bookd close\ud83d\udfe2\ud83d\udfe2"}},{"update_id":568022213,
"channel_post":{"message_id":437,"chat":{"id":-1001436032340,"title":"FORTUNA","type":"channel"},"date":1588900321,"reply_to_message":{"message_id":435,"chat":{"id":-1001436032340,"title":"FORTUNA","type":"channel"},"date":1588893671,"text":"A\nGold buy 1713.3\nSl 1702 \nTp open\nSwing trade"},"text":"Amazon\n60 pips bookd close\ud83d\udfe2\ud83d\udfe2"}},{"update_id":568022214,
"channel_post":{"message_id":438,"chat":{"id":-1001436032340,"title":"FORTUNA","type":"channel"},
MQL4 code:
int GetUpdates()
{
if(m_token==NULL)
return(ERR_TOKEN_ISEMPTY);
string out;
string url=StringFormat("%s/bot%s/getUpdates",TELEGRAM_BASE_URL,m_token);
string params=StringFormat("offset=%d",m_update_id);
//---
int res=PostRequest(out,url,params,WEB_TIMEOUT);
if(res==0)
{
Print(StringDecode(out));
//--- parse result
CJAVal js(NULL,jtUNDEF);
bool done=js.Deserialize(out);
if(!done)
return(ERR_JSON_PARSING);
bool ok=js["ok"].ToBool();
if(!ok)
return(ERR_JSON_NOT_OK);
CCustomMessage msg;
int total=ArraySize(js["result"].m_e);
for(int i=0; i<total; i++)
{
CJAVal item=js["result"].m_e[i];
//---
msg.update_id=item["update_id"].ToInt();
//---
msg.message_id=item["message"]["message_id"].ToInt();
msg.message_date=(datetime)item["message"]["date"].ToInt();
//---
msg.message_text=item["message"]["text"].ToStr();
printf((datetime)item["message"]["date"].ToInt());
msg.message_text=StringDecode(msg.message_text);
//---
msg.from_id=item["message"]["from"]["id"].ToInt();
msg.from_first_name=item["message"]["from"]["first_name"].ToStr();
msg.from_first_name=StringDecode(msg.from_first_name);
msg.from_last_name=item["message"]["from"]["last_name"].ToStr();
msg.from_last_name=StringDecode(msg.from_last_name);
msg.from_username=item["message"]["from"]["username"].ToStr();
msg.from_username=StringDecode(msg.from_username);
//---
msg.chat_id=item["message"]["chat"]["id"].ToInt();
msg.chat_first_name=item["message"]["chat"]["first_name"].ToStr();
msg.chat_first_name=StringDecode(msg.chat_first_name);
msg.chat_last_name=item["message"]["chat"]["last_name"].ToStr();
msg.chat_last_name=StringDecode(msg.chat_last_name);
msg.chat_username=item["message"]["chat"]["username"].ToStr();
msg.chat_username=StringDecode(msg.chat_username);

I tried to use the same library, it seems to have a bug with parsing arrays. That is why I used https://www.mql5.com/en/code/11134. It has a disadvantage: you need to delete all the objects, unfortunately; as a result there's plenty of code. But at least it works.
Seems your json has incorrect format, I used to cut it a little.
#include <json1.mqh> //modified version, https://www.mql5.com/en/forum/28928/page5#comment_15766620
//const string post_result=
//{"ok":true,"result":[
//{"update_id":568022212,"channel_post":{"message_id":436,"chat":{"id":-1001436032340,"title":"FORTUNA","type":"channel"},
//"date":1588899840,"reply_to_message":{"message_id":372,"chat":{"id":-1001436032340,"title":"FORTUNA","type":"channel"},
//"date":1588838583,"text":"A\nGbpusd buy now 1.2360\nSl 1.2280 \nTp open\n","entities":[{"offset":52,"length":11,"type":"mention"}]},"text":"A\n42 pips bookd close\ud83d\udfe2\ud83d\udfe2"}},
//{"update_id":568022213,"channel_post":{"message_id":437,"chat":{"id":-1001436032340,"title":"FORTUNA","type":"channel"},
//"date":1588900321,"reply_to_message":{"message_id":435,"chat":{"id":-1001436032340,"title":"FORTUNA","type":"channel"},
//"date":1588893671,"text":"A\nGold buy 1713.3\nSl 1702 \nTp open\nSwing trade"},"text":"Amazon\n60 pips bookd close\ud83d\udfe2\ud83d\udfe2" }}]}
//;
void function()
{
//---
const string post_result=getContent();//ok, you managed to get some string here
JSONParser *parser=new JSONParser();
JSONValue *value=parser.parse(post_result);
delete(parser);
if(CheckPointer(value)==POINTER_INVALID)
{
printf("%i %s: error!",__LINE__,__FILE__);
delete(value);
return;
}
JSONObject *obj=(JSONObject*)value;
const bool isSuccess=obj.getBool("ok");
printf("%i %s: isSuccess=%d",__LINE__,__FILE__,isSuccess);
if(!isSuccess)return;
JSONValue *resultValue=obj.getValue("result");
if(CheckPointer(resultValue)!=POINTER_INVALID)
{
JSONArray *resultValueArray=resultValue;
for(int i=0;i<resultValueArray.size();i++)
{
printf("%i %s: #%d=%s",__LINE__,__FILE__,i,resultValueArray.getObject(i).toString());
//you can work with JSONObject or with string, whatever is more convenient
parseResultLine(resultValueArray.getObject(i));
}
delete(resultValueArray);
}
delete(resultValue);
delete(obj);
}
bool parseResultLine(JSONObject *object)
{
const long update_id=object.getLong("update_id");
JSONObject *channel_post=object.getObject("channel_post");
const long message_id=channel_post.getLong("message_id");
const datetime date=(datetime)channel_post.getInt("date");
const string text=channel_post.getString("text");
printf("%i %s: id=%s, dt=%d/%s, text=%s",__LINE__,__FILE__,IntegerToString(message_id),(int)date,TimeToString(date),text);
JSONObject *chat=channel_post.getObject("chat");
const long chat_id=chat.getLong("id");
const string chat_title=chat.getString("title");
printf("%i %s: chat-> id=%I64d title=%s type=%s",__LINE__,__FILE__,chat_id,chat_title,chat.getString("type"));
JSONObject *reply=channel_post.getObject("reply_to_message");
printf("%i %s: replied: id=%s, date=%s",__LINE__,__FILE__,string(reply.getLong("message_id")),TimeToString(reply.getInt("date")));
return true;
}

Related

Quick way to extract the infomation from .xml files to the object

I am starter and right now I am trying to extract the key information from a .xml file then load them to an object of my class, for example:
Here are some information in .xml file:
<row Id="17" Phone="12468" Address="Bos" />
<row Id="242" Phone="98324" Address="Chi" Age="30"/>
<row Id="157" Phone="23268" Age="25" />
<row Id="925" Phone="54325" Address="LA" />
And my class would be:
class worker{
string ID;
string Phone;
string Address;
string Age;
}
I know the infomation would be various and if there is not that infomation of that line, we put ""(empty string) in it as return. And I know the infomation are given in the same order of the fields in class. I try to implement a function, let says extractInfo(const string& line, const string &key)
//#line: the whole line read from .xml
//#key: it would be "Id:"", "Phone:"", "Address:"" or "Age:"", so that I could reach the
// previous index of the infomation that I could extract.
extractInfo(const string& line, const string &key){
int index = line.find(key);
if(index == -1) return "";
int start = index + key.length(); //to reach the start quote
int end = start;
while(line[end] != '"'){ //to reach the end quote
end++;
}
return line.substr(start, end - start);
}
int main(){
...// for each line read from .xml, I build a new object of class worker and filling the field
worker.Id = extraInfo(line, "Id:\"");
worker.Phone = extraInfo(line, "Phone:\"");
...//etc.
...//then work on other manipulation
return 0;
}
My question are, is there any way that I could read and load the infomation from xml much more quickly through other APL or functions? That is, is there any way for me to improve this function when the .xml is a huge file with TBytes? And, is there any way that I can use less memory to, for example, find the oldest worker then print out? I know it's tough for me and I still try hard on it!
Thank all the ideas and advice in advance!
You can parse XML with existing XML parsing libraries, such as rapidxml, libxml2, etc.
Please note that for huge XML, since it need read all XML content to create the DOM tree, so the DOM method is not really suitable. you can use libxml2's xmlreader to parse each node one by one.
libxml2 xml reader
static void
streamFile(const char *filename) {
xmlTextReaderPtr reader;
int ret;
reader = xmlReaderForFile(filename, NULL, 0);
if (reader != NULL) {
ret = xmlTextReaderRead(reader);
while (ret == 1) {
const xmlChar *name = xmlTextReaderConstName(reader);
if(xmlStrEqual(BAD_CAST "row", name)) {
const xmlChar *id = xmlTextReaderGetAttribute(reader, "Id");
const xmlChar *phone = xmlTextReaderGetAttribute(reader, "Phone");
// you code here...
xmlFree(id);
xmlFree(phone);
}
ret = xmlTextReaderRead(reader);
}
xmlFreeTextReader(reader);
if (ret != 0) {
fprintf(stderr, "%s : failed to parse\n", filename);
}
} else {
fprintf(stderr, "Unable to open %s\n", filename);
}
}
And, If your XML format is always like above, you can also use std::regex_search to handle it
https://en.cppreference.com/w/cpp/regex/regex_search
#include <iostream>
#include <string>
#include <regex>
int main()
{
std::string str = R"(<row Id="17" Phone="12468" Address="Bos" />)";
std::regex regex("(\\w+)=\"(\\w+)\"");
// get all tokens
std::smatch result;
while (std::regex_search(str, result, regex))
{
std::cout << result[1] << ": " << result[2] << std::endl;
str = result.suffix().str();
}
}

catch error from blank xml file

I am using C++ and I read in and parse an XML file. If the file does not exist it gets created and if it does exist and has correctly formatted data in it there no errors. My problem is when I try to read in a blank XML file (completely blank not even the header) my program crashes as it tries to process the data from the XML file. I need to be able to catch the error before that happen so I can wright the correct settings back to the file or remove the file and make a new one. Is there a way to do this such as checking to see if a node is NULL? And how would I do it?
class XmlDomErrorHandler : public HandlerBase
{
public:
void fatalError(const SAXParseException &exc) {
printf("Fatal parsing error at line %d\n", (int)exc.getLineNumber());
exit(-1);
}
};
void XmlParentNode::refreshAllNodes()
{
m_children.clear();
m_childrernByName.clear();
for(int j =0; j < getInnerParentNode()->getChildNodes()->getLength(); ++j) //crashes here on first pass if xmlfile is empty
{
//code that is never reached
}
XmlDocument::XmlDocument(String name):
XmlParentNode(),
m_pOwner(NULL)
{
if(XmlManager::Get()->initialize())
{
XMLCh* xmlName = XMLString::transcode(StringA(name));
setInnerNode(XmlManager::Get()->getDOMImplementation()->createDocument(NULL, xmlName , NULL));
XMLString::release(&xmlName);
}
}
XmlDocument::XmlDocument(FilePath path):
XmlParentNode(),
m_pOwner(NULL)
{
XMLCh* xmlPath = XMLString::transcode(StringA(path.ToString()));
XercesDOMParser* parser = new XercesDOMParser();
parser->parse(xmlPath);
if(!parser)
{
ErrorHandler* errorHandler = (ErrorHandler*) new XmlDomErrorHandler();
parser->setErrorHandler(errorHandler);
}
DOMDocument* p_Doc= parser->getDocument();
m_pOwner = parser;
XMLString::release(&xmlPath);
if (p_Doc!= NULL)
{
setInnerNode(p_Doc);
refreshAllNodes();
}
}
Sadly I can't stop people for editing the XML file as users need access to it so they can edit settings.
I am new to XML so maybe I'm missing something easy.
I have been using XML Parsing with DOM and a guide I found here
I solved the problem.
I added a check before the for loop that crashed.
if(getInnerParentNode() && getInnerParentNode()->getChildNodes())
{
for(int j =0; j < getInnerParentNode()->getChildNodes()->getLength(); ++j) //was crashing here on first pass if xmlfile is empty
{
//code that was not being reached
}
}

How to find a string is Json or not using Jansson?

I am using Jansson.
bool ConvertJsontoString(string inputText, string& OutText)
{
/* Before doing anything I want to check
if the inputText is a valid json string or not */
}
Why don't you read the documentation where it clearly states:
json_t *json_loads(const char *input, size_t flags, json_error_t *error)
Return value: New reference.
Decodes the JSON string input and returns the array or object it contains,
or NULL on error, in which case error is filled with information about the error.
flags is described above.
Also they even provide an example on how to use this:
root = json_loads(text, 0, &error);
free(text);
if(!root)
{
fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
return 1;
}

Loading Local Content in Awesomium

I have written a local DataSource because from what I know there are none included in Awesomium, but the thing is that it request everything from the data source html, images etc
And I have no clue on how I should load all types of mime formats.
My current code only supports html/text, where I load the file into binary and send as a response. This does not work for images.
Does anybody know where I should go on from here?
class LocalDataSource :
public Awesomium::DataSource
{
public:
LocalDataSource() { }
virtual ~LocalDataSource() { }
virtual void OnRequest(int request_id, const Awesomium::WebString& path)
{
std::string filepath = Awesomium::ToString(path).insert(0, "./");
std::basic_ifstream<char> is(filepath, std::ios_base::in | std::ios_base::binary);
if (is)
{
is.seekg(0, is.end);
int length = is.tellg();
is.seekg(0, is.beg);
char *buffer = new char[length + 1];
is.read(buffer, length);
buffer[length] = '\0';
is.close();
SendResponse(request_id, strlen(buffer), (unsigned char*)buffer, Awesomium::WSLit("text/html"));
delete[] buffer;
}
else
{
// Error
}
}
};
EDIT:
for now I will load the file relative to the executable and not use DataSource's.
I know this is old, but it was relevant to me, I fixed this the same way as Steven did, I will post the C++ code I used:
bool ResInterceptor::OnFilterNavigation(int origin_process, int origin_routing_id, const Awesomium::WebString& method, const Awesomium::WebURL& url, bool is_main_frame)
{
return false;
}
Awesomium::ResourceResponse* ResInterceptor::OnRequest(Awesomium::ResourceRequest* request)
{
bool isAsset = std::strcmp(ToString(request->url().scheme()).c_str(), "asset")==0;
bool isFile = std::strcmp(ToString(request->url().scheme()).c_str(), "file")==0;
if(!isAsset && !isFile)
{
//if it is neither of these we "may" still intercept the call, this allows for offline-online versions to work
return Awesomium::ResourceInterceptor::OnRequest(request);
}
if(isAsset)
{
//Blah blah, do whatever
}
else if(isFile)
{
//Blah blah, same
}
//As you can see this isn't very, but it worked for my purposes
std::string contentpath = "E:/Location/of/files" + ToString(request->url().path());
Awesomium::WebString datatype;
std::string filename = Awesomium::ToString(request->url().filename());
//I still want to check for the correct mime type
if (has_suffix(filename, ".html")) datatype = Awesomium::WSLit("text/html");
else if(has_suffix(filename, ".js")) datatype = Awesomium::WSLit("text/javascript");
else if(has_suffix(filename, ".css")) datatype = Awesomium::WSLit("text/css");
else if(has_suffix(filename, ".swf")) datatype = Awesomium::WSLit("application/x-shockwave-flash");
else if(has_suffix(filename, ".zip")) datatype = Awesomium::WSLit("application/zip");
else if(has_suffix(filename, ".txt")) datatype = Awesomium::WSLit("text/plain");
else if(has_suffix(filename, ".text")) datatype = Awesomium::WSLit("text/plain");
else if(has_suffix(filename, ".png")) datatype = Awesomium::WSLit("image/png");
else if(has_suffix(filename, ".jpeg")) datatype = Awesomium::WSLit("image/jpeg");
else if(has_suffix(filename, ".jpg")) datatype = Awesomium::WSLit("image/jpeg");
else if(has_suffix(filename, ".webm")) datatype = Awesomium::WSLit("video/webm");
else if(has_suffix(filename, ".mp4")) datatype = Awesomium::WSLit("video/mp4");
else if(has_suffix(filename, ".ogv")) datatype = Awesomium::WSLit("video/ogg");
else if(has_suffix(filename, ".flv")) datatype = Awesomium::WSLit("video/flv");
if(!datatype.IsEmpty())
{
FILE * pFile;
long lSize;
unsigned char * buffer;
size_t result;
pFile = fopen ( contentpath.c_str() , "rb" );
if (pFile!=NULL)
{
// obtain file size:
fseek (pFile , 0 , SEEK_END);
lSize = ftell (pFile);
rewind (pFile);
// allocate memory to contain the whole file:
buffer = (unsigned char*) malloc (sizeof(unsigned char)*lSize);
if (buffer == NULL) {fputs ("Memory error",stderr); exit (2);}
// copy the file into the buffer:
result = fread (buffer,1,lSize,pFile);
if (result != lSize) {fputs ("Reading error",stderr); exit (3);}
//This is where the magic happens!!
return Awesomium::ResourceResponse::Create(lSize, buffer, datatype);
// terminate
fclose (pFile);
free (buffer);
}
else
{
//send this off to the default request handler instead of it being a local file
return Awesomium::ResourceInterceptor::OnRequest(request);
}
}else
{
//send this off to the default request handler instead of it being a local file
return Awesomium::ResourceInterceptor::OnRequest(request);
}
}
//Support function
bool ResInterceptor::has_suffix(const std::string &str, const std::string &suffix)
{
return str.size() >= suffix.size() &&
str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
}
as for how I hooked it up, I simply added this line of code:
_web_core = WebCore::Initialize(config);
_web_core->set_resource_interceptor(new ResInterceptor());
This took me a whole night to nail down all because I was passing in a pointer with a variable and not using the "new" keyword directly! I got it now at least!
also note, I tried the exact same code above inside the LocalDataSource and it didn't work for anything except the text files, so I think there is a bug in there, good news is, this works the exact same way, but you get more control over every file request.
Thank you Steven for all the great reference code!
The easy way is to send the contents of a file without sweating mime type detection is to use the static method static ResourceResponse* Awesomium::ResourceResponse::Create.
From the Awesomium docs:
Create a ResourceResponse from a file on disk.
I couldn't figure out a way to map ResourceResponse::Create to DataSource::SendResponse.
As a workaround, you could rewrite your data source as an IResourceInterceptor instead of a DataSource. I wrote up a detailed example in C# on how to use http:// scheme instead of the custom asset:// scheme for embedded resources It should be pretty straightforward to translate the C# to C++. Below is an edited down version of my post (not tested).
using System;
using System.IO;
using System.Reflection;
using Awesomium.Core;
namespace MyApp
{
public class ResourceInterceptor : IResourceInterceptor
{
/// <summary>
/// Intercepts any requests for the EmbeddedResourceDomain base Uri,
/// and returns a response using the embedded resource in this app's assembly/DLL file
/// </summary>
public virtual ResourceResponse OnRequest(ResourceRequest request)
{
ResourceResponse response = null;
string resourceName;
string filePath;
filePath = String.Concat("./", request.Url.AbsolutePath);
filePath = Path.GetFullPath(resourceName.Replace('/', Path.DirectorySeparatorChar));
// cache the resource to a temp file if
if (File.Exists(filePath))
{
response = ResourceResponse.Create(filePath);
}
return response;
}
/// <summary>
/// Optionally blocks any web browser requests by returning true. Not used.
/// </summary>
/// <remarks>
/// This method can implement a whitelist of allowed URLs here by
/// returning true to block any whitelist misses
/// </remarks>
public virtual bool OnFilterNavigation(NavigationRequest request)
{
return false;
}
}
}
Another option might be to hack the HTML content to inject a <base href="file:///c:/my/bin/path" /> element inside the <head> of the document. You would need to modify the href attribute value before loading the content. This may be more work than it is worth.

another C++ equivalent for the followingtwo lines of code

Sorry for such a vague title.
Basically, I am trying to hack a function to suit my needs. But lately I have been working a lot on python and my c++ is bit rusty.
So earlier my function took a
int func(FILE *f)
{ .....
if (fgets(line, MM_MAX_LINE_LENGTH, f) == NULL)
return MM_PREMATURE_EOF;
if (sscanf(line, "%s %s %s %s %s", banner, mtx, crd, data_type,
storage_scheme) != 5)
return MM_PREMATURE_EOF;
}
now instead of this I am directly inputing the string data
int func(std::string *data)
{ .....
// how should I modify this if statment..I want to parse the same file
// but instead it is in form of one giant string
}
Thanks
You can use the same code, just convert the data in the std::string into a C string.
sscanf(data->c_str(), "%s %s %s %s %s", //...);
However, you should consider passing in a const reference, since you are probably not planning on modifying your input data:
int func(const std::string &data) {
//...
if (sscanf(data.c_str(), //...)) {
//...
}
}