I'm fetching 43182 chars long JSON from remote REST API using following code snippet:
string result_;
curl_easy_setopt(curlGET_, CURLOPT_TIMEOUT, CURL_TIMEOUT);
curl_easy_setopt(curlGET_, CURLOPT_URL, url.c_str());
curl_easy_setopt(curlGET_, CURLOPT_VERBOSE, CURL_DEBUG_VERBOSE);
curl_easy_setopt(curlGET_, CURLOPT_WRITEDATA, &result_);
curl_easy_setopt(curlGET_, CURLOPT_WRITEFUNCTION, WriteCallback);
static size_t WriteCallback(void *response,
size_t size,
size_t nmemb,
void *userp) noexcept
{
(static_cast<string*>(userp))->append(static_cast<char*>(response));
return size * nmemb;
};
curlStatusCode_ = curl_easy_perform(curlGET_);
What I get in result_ is complete JSON but with newline character after character 31954:
If I fetch same JSON in browser or command-line curl there is no newline character. How to fix this problem for "arbitrarily" long JSON or other generic response?
From CURL document of the write callback:
The data passed to this function will not be zero terminated!
The response in WriteCallback is not necessarily null terminated. So calling append by just casting it to char* will invoke undefined behavior. You have to pass the amount of data too in append.
(static_cast<string*>(userp))->append(static_cast<char*>(response), size * nmemb)
Related
I am using following code to download data from an url to memory (stream). Around 2% chance, the size of the stream is zero. I can download proper data from the same failing url if I try it another time. I am not sure if this is a network issue, CPU usage issue, or it's just the code not covering some corner cases. Please advice. Thanks!
static size_t write_data(char *ptr, size_t size, size_t nmemb, void *userdata)
{
std::vector<uchar> *stream = (std::vector<uchar>*)userdata;
size_t count = size * nmemb;
stream->insert(stream->end(), ptr, ptr + count);
return count;
}
static void CurlUrl(const char* img_url, std::vector<uchar>* stream) {
CURL *curl = curl_easy_init(); // curl_global_init is called eleswhere.
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(curl, CURLOPT_URL, img_url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, stream);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10);
CURLcode res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
If it didn't deliver any download data into the buffer via the callback, it means that the transfer either failed or that there was exactly zero bytes to transfer.
Check the return code from curl_easy_perform() as it might actually tell you exactly what happened.
Use CURLOPT_VERBOSE to see what's going on if (1) is not enough.
Use CURLOPT_ERRORBUFFER to get a better error description if it fails if (2) is not enough.
How can you prevent / remove / stop the print/output to the command line in libcurl? For some reason, the following code works for URLs, but when I am hitting a page with an image, like http://192.168.123.123/banana.gif , then it crashes:
CURL *session;
session = curl_easy_init();
curl_easy_setopt(session, CURLOPT_URL, "http://192.168.123.123/banana.gif");
CURLcode curl_code = curl_easy_perform (session);
long http_code = 0;
curl_easy_getinfo(session, CURLINFO_RESPONSE_CODE, &http_code);
This is because you did not set the CURLOPT_WRITEDATA option:
The internal CURLOPT_WRITEFUNCTION will write the data to the FILE * given with this option, or to stdout if this option hasn't been set.
What you can do if you decide to completely ignore the response data is write it to /dev/null:
/* ... */
FILE *devnull = fopen("/dev/null", "w+");
curl_easy_setopt(session, CURLOPT_WRITEDATA, devnull);
CURLcode curl_code = curl_easy_perform(session);
fclose(devnull);
/* ... */
Another alternative is to use a NOOP write function:
curl_easy_setopt(session, CURLOPT_WRITEFUNCTION, noop_cb);
Where the write function simply returns the number of received bytes:
size_t noop_cb(void *ptr, size_t size, size_t nmemb, void *data) {
return size * nmemb;
}
The below code is to get response from a server using wsdl, here the problem is curl returns response but am unable to print it.
Error:
Failed writing body
Failed writing data
#include<stdio.h>
#include<string.h>
#include"../include/curl.h"
size_t write_data(void *ptr, size_t size, size_t count, void *stream)
{
/* ptr - your string variable.
stream - data chuck you received */
printf("%.*s", size, (char*)stream);
}
int main()
{
int res=0,i=0;
char buffer[4098]="",buff[128]="",buf[256]="",buf7[30]="",buf6[30]="",buf5[30]="";
char machineid[]="SUBANI";
char filename1[50]="";
int refno=0,paymode=0,taxtype=0;
FILE *fbc;
memset(filename1,0,sizeof(filename1));
sprintf(filename1,"/mnt/jffs2/Response_Details1.xml");
lk_dispclr();
lk_disptext(1,0,(unsigned char *)"Sending Request",0);
lk_disptext(2,0,(unsigned char *)"Please Wait",0);
memset(buffer,0,sizeof(buffer));
sprintf(buffer,"<?xml version=\"1.0\" encoding=\"utf-8\"?>\
<soap:Envelope xmlns:soap=\"http://www.w3.org/2003/05/soap-envelope\" xmlns:log=\"http://wsdlclassess.application.sims.test.com\">\
<soap:Header>\
</soap:Header>\
<soap:Body>\
<log:loginMethod>\
<log:loginid>%s</log:loginid>\
<log:password>%s</log:password>\
</log:loginMethod>\
</soap:Body>\
</soap:Envelope>","raja","test");
res=GET_FILE1(buffer,filename1);
return 0;
}
int GET_FILE1(char *buffer,char *filename)
{
CURL *curl;
CURLcode res;
struct curl_slist *headers = NULL;
FILE *out_fd = (FILE *) 0;
char errorbuf[300] = "",tmpbuff[128]="";
char errmsg[256];
int Timeout=120; //Default timeout is = 2 mins
int buffer_size = 0;
char urlbuff[256]="";
char mstr[10240];
memset(urlbuff,0,sizeof(urlbuff));
memset(tmpbuff,0,sizeof(tmpbuff));
buffer_size = strlen(buffer);
strcpy(tmpbuff,"http://10.10.1.111:8081/test_server/services/application?wsdl");
tmpbuff[strlen(tmpbuff)]='\0';
curl = curl_easy_init();
if(curl)
{
out_fd = fopen (filename, "w");
curl_easy_setopt(curl, CURLOPT_FILE, out_fd);
printf("%s:Sign-In Request\n", __func__);
headers = curl_slist_append(headers, "Content-type:application/soap+xml; charset=utf-8; action=\"http://wsdlclassess.application.sims.test.com/loginMethod\"");
curl_easy_setopt(curl, CURLOPT_URL, tmpbuff);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, mstr);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, buffer_size);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buffer);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, Timeout);
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER,errmsg);
printf("The Server%s:Performing Transaction.....\n",__func__);
res = curl_easy_perform(curl);
printf("res=after culreasey perform%d\n",res);
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
printf("\nerrorbuf:%s\n",errmsg);
fclose(out_fd);
if(CURLE_OK != res)
{
puts("error occured is\n" );
//ppp_close();
return -1;
}
}
return 0;
}
The error is that you don't return the correct value from the function, in fact you don't return anything.
Also, the data provided to the function is actually the first ptr argument.
I agree that the documentation is not very clear, but it says:
The size of the data pointed to by ptr is size multiplied with nmemb, it will not be zero terminated.
The above line (emphasis mine) tells you that the data is in ptr which is the first argument in the function declaration provided in the documentation.
The documentation also states:
Return the number of bytes actually taken care of. If that amount differs from the amount passed to your function, it'll signal an error to the library. This will abort the transfer and return CURLE_WRITE_ERROR.
You don't return a value from the function, and so you have undefined behavior with a seemingly random value being returned causing the whole operation to fail. To fix this you should return size * count.
You also uses size to print the string, which is the size of the underlying type used (probably 1), your count variable is the number of characters read by CURL. To be fully working, without invoking more undefined behavior (since the data is not terminated) you should call printf like:
printf("%*.*s", size * count, size * count, ptr);
I am trying to read the content of a PHP / HTML file on a remote web server using C++, but haven't found a way to do it. I want to pass GET statements to it, so http://example.com/login.php?user=abc&password=def.
How would I do it?
Your best bet is to use an external library. libcurl is popular and fairly easy to use.
Here's a simple example, you need to add error checking though:
string data;
CURL *curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, url_.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWrite);
curl_easy_perform(curl);
Your callback would look something like this:
size_t curlWrite(void *ptr, size_t size, size_t nmemb, void *usrPtr)
{
size_t bytes = size * nmemb;
string *data = static_cast<string *>(usrPtr);
data->append(static_cast<const char *>(ptr), bytes);
return bytes;
}
You can add your GET parameters on the end of the URL.
Take a look at the following code
static size_t reader(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t retcode = fread(ptr, size, nmemb, stream);
cout << "*** We read " << retcode << " bytes from file" << endl;
return retcode;
}
void upload() { //upload() is called from ouside
FILE *pFile;
pFile = fopen("map.txt" , "r");
struct stat file_info;
stat("map.txt", &file_info);
size_t size = (size_t)file_info.st_size;
uploadFile(pFile, size);
}
bool uploadFile(void* data, size_t datasize) {
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if (curl) {
char *post_params = ...;
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_params);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long) strlen(post_params));
curl_easy_setopt(curl, CURLOPT_READFUNCTION, reader);
curl_easy_setopt(curl, CURLOPT_READDATA, data);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t) datasize);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
return true;
}
When the code is executed, the following is outputed
*** We read 490 bytes from file
*** We read 0 bytes from file
after that the app does nothing (even not exiting).
Can someone point out at what's wrong here?
Will be grateful for any help!!!
There's some serious confusions shown in this code. Let me try to explain:
CURLOPT_UPLOAD - this will ask libcurl to PUT the file when the protocol of choice is HTTP
CURLOPT_POSTFIELDS - tells libcurl to POST the data that is provided in the additional argument (which has the size set with CURLOPT_POSTFIELDSIZE)
CURLOPT_READFUNCTION - provides libcurl an alternative way to get data than CURLOPT_POSTFIELDS to allow a POST that reads the data from a file. When using CURLOPT_UPLOAD this is the only way to provide data.
So in the end the questions left for you are:
Do you want PUT or POST?
Do you want to provide the data as a string or do you want it provided with a callback?