The code below is a test for the CURL C API . The problem is that the callback function write_callback is never called. Why ?
/** compilation: g++ source.cpp -lcurl */
#include <assert.h>
#include <iostream>
#include <cstdlib>
#include <cstring>
#include <cassert>
#include <curl/curl.h>
using namespace std;
static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp)
{
std::cerr << "CALLBACK WAS CALLED" << endl;
exit(-1);
return size*nmemb;
}
static void test_curl()
{
int any_data=1;
CURLM* multi_handle=NULL;
CURL* handle_curl = ::curl_easy_init();
assert(handle_curl!=NULL);
::curl_easy_setopt(handle_curl, CURLOPT_URL, "http://en.wikipedia.org/wiki/Main_Page");
::curl_easy_setopt(handle_curl, CURLOPT_WRITEDATA, &any_data);
::curl_easy_setopt(handle_curl, CURLOPT_VERBOSE, 1);
::curl_easy_setopt(handle_curl, CURLOPT_WRITEFUNCTION, write_callback);
::curl_easy_setopt(handle_curl, CURLOPT_USERAGENT, "libcurl-agent/1.0");
multi_handle = ::curl_multi_init();
assert(multi_handle!=NULL);
::curl_multi_add_handle(multi_handle, handle_curl);
int still_running=0;
/* lets start the fetch */
while(::curl_multi_perform(multi_handle, &still_running) ==
CURLM_CALL_MULTI_PERFORM );
std::cerr << "End of curl_multi_perform."<< endl;
//cleanup should go here
::exit(EXIT_SUCCESS);
}
int main(int argc,char** argv)
{
test_curl();
return 0;
}
Many thanks
Pierre
You need to check the value of still_running and call curl_multi_perform() again if there are still pending operations.
Simple example:
int still_running=0;
/* lets start the fetch */
do {
while(::curl_multi_perform(multi_handle, &still_running) ==
CURLM_CALL_MULTI_PERFORM);
} while (still_running);
Related
Closed. This question is not reproducible or was caused by typos. It is not currently accepting answers.
This question was caused by a typo or a problem that can no longer be reproduced. While similar questions may be on-topic here, this one was resolved in a way less likely to help future readers.
Closed last month.
Improve this question
I'm trying to create c++ code that download data from some URLs, but it's throwing a write access violation:
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#ifndef WIN32
#endif
#include <curl/curl.h>
#include <string>
static const char* urls[] = {
"http://www.example.com",
"http://www.example1.com",
};
#define MAX_PARALLEL 10 /* number of simultaneous transfers */
#define NUM_URLS sizeof(urls)/sizeof(char *)
static size_t write_cb(void* ptr, size_t size, size_t nmemb, void* buffer)
{
((std::string*)buffer)->append((char*)ptr, nmemb);
return nmemb;
}
static void add_transfer(CURLM* cm, int i, int* left)
{
CURL* eh = curl_easy_init();
curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, write_cb);
curl_easy_setopt(eh, CURLOPT_URL, urls[i]);
curl_easy_setopt(eh, CURLOPT_PRIVATE, urls[i]);
curl_easy_setopt(eh, CURLOPT_WRITEDATA, &write_cb);
curl_easy_setopt(eh, CURLOPT_VERBOSE, 1L);
curl_multi_add_handle(cm, eh);
(*left)++;
}
int main(void)
{
CURLM* cm;
unsigned int transfers = 0;
int msgs_left = -1;
int left = 0;
curl_global_init(CURL_GLOBAL_ALL);
cm = curl_multi_init();
/* Limit the amount of simultaneous connections curl should allow: */
curl_multi_setopt(cm, CURLMOPT_MAXCONNECTS, (long)MAX_PARALLEL);
for (transfers = 0; transfers < MAX_PARALLEL && transfers < NUM_URLS;
transfers++)
add_transfer(cm, transfers, &left);
do {
int still_alive = 1;
curl_multi_perform(cm, &still_alive);
CURLMsg* msg;
int queued;
CURLMcode mc = curl_multi_perform(cm, &still_alive);
if (cm)
/* wait for activity, timeout or "nothing" */
mc = curl_multi_poll(cm, NULL, 0, 1000, NULL);
if (mc)
break;
do {
msg = curl_multi_info_read(cm, &queued);
if (msg) {
if (msg->msg == CURLMSG_DONE) {
/* a transfer ended */
fprintf(stderr, "Transfer completed\n");
}
}
} while (msg);
if (left)
curl_multi_wait(cm, NULL, 0, 1000, NULL);
} while (left);
curl_multi_cleanup(cm);
curl_global_cleanup();
return EXIT_SUCCESS;
}
Its crashing on the line:
_Mypair._Myval2._Mysize = _Old_size + _Count;
The full error message is:
Exception thrown: write access violation.
this was 0x7FF7941D39D0.
How can I make this code download each Url data without any error?
In your write_cb() callback, you are expecting the buffer parameter to point at a std::string object, but in add_transfer() you are setting CURLOPT_WRITEDATA to point at write_cb itself rather than at a std::string object.
Try something more like this instead:
struct url_info {
const char* url;
std::string data;
};
static url_info urls[] = {
{"http://www.example.com", ""},
{"http://www.example1.com", ""}
};
static const int NUM_URLS = sizeof(urls)/sizeof(urls[0]);
static size_t write_cb(void* ptr, size_t size, size_t nmemb, void* buffer)
{
size_t result = size * nmemb;
static_cast<std::string*>(buffer)->append(static_cast<char*>(ptr), result);
return result;
}
static void add_transfer(CURLM* cm, int i, int* left)
{
...
curl_easy_setopt(eh, CURLOPT_URL, urls[i].url);
curl_easy_setopt(eh, CURLOPT_WRITEDATA, &urls[i].data);
curl_easy_setopt(eh, CURLOPT_PRIVATE, &urls[i]);
...
}
int main(void)
{
...
while ((msg = curl_multi_info_read(cm, &queued)) != NULL) {
if (msg->msg == CURLMSG_DONE) {
/* a transfer ended */
url_info *info;
curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, reinterpret_cast<char**>(&info));
// use info->url and info->data as needed...
std::cerr << "Transfer completed from " << info->url << ", bytes received: " << info->data.size() << "\n";
}
}
...
}
#pragma once
#ifndef __CURL_CURL_H
#include "curl.h"
#endif
#ifndef __CURL_EASY_H
#include "easy.h"
#endif
#include <stdint.h>
#include <memory>
#include <string>
namespace CommUnit
{
enum ERR_PROXY
{
ERR_CURL_INIT_FAILED = 0xA0,
ERR_SET_PROXY_FAILED = 0xA1,
};
class MyProxy
{
public:
static MyProxy & GetInstance() //Meyers' Singlton
{
static MyProxy ProxySigleton;
return ProxySigleton;
}
public:
/*
* #bref:Get request
* #param[in] sUrl:Access URL
* #param[in] sProxyIp:Proxy IP
* #param[in] uProxyPort:Proxy Port
* #param[in] uTimeOut:Time out
* #param[in] isSSL:HTTPS true,else false
* #param[out] sRetContent:Return the URL content
*/
uint32_t Get(const std::string &sUrl,
const std::string& sProxyIp,
uint32_t uProxyPort,
uint32_t uTimeOut,
bool isSSL,
std::string &sRetContent);
private:
MyProxy(); //Constructor hidden
MyProxy(MyProxy const &); //Copy-Constructor hidden
MyProxy & operator= (MyProxy const &); //Assign operator hidden
~MyProxy(); //Destructor hidden
inline void _setCurlopt(CURL *pCurl,
const std::string &sUrl,
std::string &sWriterData,
const uint32_t uTimeOut,
bool isSSL);
//Callback function, write data to writerData
static int Writer(char *data,
uint32_t size,
uint32_t nmemb,
std::string *writerData);
private:
std::string m_sErrMsg;
static char s_ErrBuffer[CURL_ERROR_SIZE];
static const uint32_t m_MAXBUF = 2 * 1024 * 1024 - 128;
};
}
//////////////////////////////////////////////////////////////////////////
#include <iostream>
#include <string>
#include "MyProxy.h"
#include "Log.h"
#include <curl.h>
using namespace CommUnit;
char MyProxy::s_ErrBuffer[CURL_ERROR_SIZE] = { 0 };
MyProxy::MyProxy(void)
{
CURLcode oCURLcode = curl_global_init(CURL_GLOBAL_ALL);
if (oCURLcode != CURLE_OK)
{
Log("ERR: %s curl_init failed!", __func__);
}
}
MyProxy::~MyProxy(void)
{
curl_global_cleanup();
}
uint32_t MyProxy::Get(const std::string &sUrl,
const std::string& sProxyIp,
uint32_t uProxyPort,
uint32_t uTimeOut,
bool isSSL,
std::string &sRetContent)
{
sRetContent.clear();
CURL *pCurl = curl_easy_init();
CURLcode oCURLcode;
if (nullptr == pCurl)
{
Log("ERR: %s curl_easy_init failed!", __func__);
return ERR_CURL_INIT_FAILED;
}
_setCurlopt(pCurl, sUrl, sRetContent, uTimeOut, isSSL);
if (0 == sProxyIp.length()|| 0 == uProxyPort)
{
Log("ERR: %s SetProxy: ProxyIp [%s], ProxyPort[%u] failed",__func__, sProxyIp.c_str(), uProxyPort);
return ERR_SET_PROXY_FAILED;
}
Log("INFO: %s SetProxy: ProxyIp [%s], ProxyPort[%u] failed", __func__, sProxyIp.c_str(), uProxyPort);
curl_easy_setopt(pCurl, CURLOPT_PROXY, sProxyIp.c_str());
curl_easy_setopt(pCurl, CURLOPT_PROXYPORT, uProxyPort);
int iTimes = 0;
while (true)
{
oCURLcode = curl_easy_perform(pCurl);
if (oCURLcode != CURLE_OK && ++iTimes < 3)
usleep(5);
else
break;
}
if (oCURLcode != CURLE_OK)
{
Log("ERR: %s curl_easy_perform failed!", __func__);
}
curl_easy_cleanup(pCurl);
return oCURLcode;
}
void MyProxy::_setCurlopt(CURL *pCurl,
const std::string &sUrl,
std::string &sWriterData,
const uint32_t uTimeOut,
bool isSSL)
{
curl_easy_setopt(pCurl, CURLOPT_ERRORBUFFER, s_ErrBuffer);
curl_easy_setopt(pCurl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(pCurl, CURLOPT_URL, sUrl.c_str());
curl_easy_setopt(pCurl, CURLOPT_TIMEOUT, uTimeOut);
Log("INFO: %s Set Url:[%s],TimeOut:[%d]", __func__, sUrl.c_str(), uTimeOut);
curl_easy_setopt(pCurl, CURLOPT_WRITEFUNCTION, MyProxy::Writer);
curl_easy_setopt(pCurl, CURLOPT_WRITEDATA, &sWriterData);
//Skip peer and hostname verification
if (isSSL)
{
curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYHOST, 0L);
}
}
int MyProxy::Writer(char *data,
uint32_t size,
uint32_t nmemb,
std::string *writerData)
{
if (writerData == nullptr)
{
Log("ERR: %s writerData is null!", __func__);
return 0;
}
int len = size * nmemb;
if ((writerData->size() + len) > m_MAXBUF)
{
Log("ERR: %s writerData size over MAXBUF!", __func__);
return 0;
}
writerData->append(data, len);
return len;
}
I want to realize a proxy with libcurl, which can get the content of given url(https). Morever, it need to be thread-safety.
But when I created 200 threads with pthreads to test my code, an segment fault occured sometimes.
How can I solve this problem?
Is there a relation with the sRetContent(std::string)?
Thanks!
Errmsg:
double free or corruption (!prev): 0x0ac72840 ***
Segmentation fault
My understanding is that libcurl is not thread-safe if you are using https (and it looks like you are) due to the fact that it is using underlying ssl libraries. See libcurl documentation and OpenSSL documentation for more info.
If your libcurl was compiled with OpenSSL for example then you have to initialize a few callback functions or you could run into issues. This is the sort of thing you need to do (compiles on Windows):
#include <curl/curl.h>
#include <openssl/crypto.h>
void win32_locking_callback(int mode, int type, const char *file, int line)
{
if (mode & CRYPTO_LOCK)
{
WaitForSingleObject(lock_cs[type],INFINITE);
}
else
{
ReleaseMutex(lock_cs[type]);
}
}
void thread_setup(void)
{
int i;
lock_cs=(HANDLE*)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(HANDLE));
for (i=0; i<CRYPTO_num_locks(); i++)
{
lock_cs[i]=CreateMutex(NULL,FALSE,NULL);
}
CRYPTO_set_locking_callback((void (*)(int,int,const char *,int))win32_locking_callback);
}
void thread_cleanup(void)
{
int i;
CRYPTO_set_locking_callback(NULL);
for (i=0; i<CRYPTO_num_locks(); i++)
CloseHandle(lock_cs[i]);
OPENSSL_free(lock_cs);
}
I always call thread_setup() after my call to curl_global_init(CURL_GLOBAL_ALL)
and then thread_cleanup() right before my call to curl_global_cleanup().
I use this sort of code with libcurl often in load test scenarios and have never run into any issues. If you continue to run into problems, it is not libcurl, but something not being done properly in your code.
libcurl is thread-safe as long as you play by the rules
**How to make a dll using libcurl. can you share me a sample code.**
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <iostream>
#include <curl/curl.h>
using namespace std; string data; typedef
#ifdef BUILDING_TESTCURL_DLL
#define TESTCURL_DLL __declspec(dllexport)
#else
#define TESTCURL_DLL __declspec(dllimport)
#endif
curl_global_init(CURL_GLOBAL_ALL); //pretty obvious
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, lpURL);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &writeCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);
curl_easy_perform(curl);
if (data == "1")
{
//printf("Successfull");
strcpy(Result, "1");
}
v5 = 1;
curl_easy_cleanup(curl);
curl_global_cleanup();
size_t TESTCURL_DLL writeCallback(char* buf, size_t size, size_t
nmemb, void* up) {
for (int c = 0; c<size*nmemb; c++) { data.push_back(buf[c]);
//printf("", buf[c]); } return size*nmemb; //tell curl how many
bytes we handled }
I run this code with MinGW: g++ -shared -o testcurl.dll testcurl.o -Wl,--out-implib,libcurldll.a
it shows error: undefined reference to '_imp_curl_easy_setopt' undefined reference to '_imp_curl_easy_perform' undefined reference to '_imp_curl_easy_init'
Please help me for developing a dll file using curl.
I am using libcurl to download serialized code and bust it open but, I get an error that looks like fstream is missing but it is very much included. I looked around but very little on the error.
Below is the error and code.
What did miss?
compile error output
g++ -g testGetprice2.cpp -o testGetprice2.o -std=gnu++11 -lcurl
testGetprice2.cpp: In function 'int getData()':
testGetprice2.cpp:45:56: error: 'ios_base' has not been declared
testGetprice2.cpp:45:72: error: 'ios_base' has not been declared
code:
#include "rapidjson/include/rapidjson/document.h"
#include <iostream>
#include <fstream>
#include <stdio.h>
#include <curl/curl.h>
#include <unistd.h>
#include <unordered_map>
#include <string>
using namespace rapidjson;
struct myData
{
std::fstream *file;
std::string *str;
};
size_t write_data(void *ptr, size_t size, size_t nmemb, myData *data)
{
size_t numBytes = size * nmemb;
if (data->file)
data->file->write((char*)ptr, numBytes);
if (data->str)
*data->str += std::string((char*)ptr, numBytes);
return numBytes;
}
//function to get coin data and perform analysis
int getData()
{
int count = 0;
//begin non terminating loop
while(true)
{
count++;
CURL *curl = curl_easy_init();
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, "http://pubapi.cryptsy.com/api.php?method=singlemarketdata&marketid=155");
std::fstream file("/home/coinz/cryptsy/myfile.txt", ios_base::out | ios_base::ate);
std::string json;
myData data;
data.file = &file;
data.str = &json;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
/* Perform the request, res will get the return code */
CURLcode res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK)
{
std::cerr << "curl_easy_perform() failed: " << curl_easy_strerror(res) << std::endl;
}
else
{
file << std::endl;
//begin deserialization
Document document;
document.Parse(json.c_str());
assert(document.HasMember("lasttradeprice"));
assert(document["hello"].IsString());
std::cout << "The Last Traded Price is = " << document["lasttradeprice"].GetString() << std::endl;
}
/* always cleanup */
curl_easy_cleanup(curl);
}
//timer for URL request. *ADUJST ME AS DESIRED*
usleep(10000000);
}
return 0;
}
//Le Main
int main(void)
{
getData();
}
ios_base is in namespace std. Add prefix std:: before ios_base.
I have data in a JSON object that I can't seem to get to.
Error:
> testGetprice3.o: testGetprice3.cpp:71: int getData(): Assertion
> `document["success"].IsString()' failed.
I have tried switching the datatypes, it only errors.
How do I correctly access my JSON object data?
#include "rapidjson/include/rapidjson/document.h"
#include <iostream>
#include <fstream>
#include <stdio.h>
#include <curl/curl.h>
#include <unistd.h>
#include <unordered_map>
#include <string>
using namespace rapidjson;
struct myData
{
std::fstream *file;
std::string *str;
};
size_t write_data(void *ptr, size_t size, size_t nmemb, myData *data)
{
size_t numBytes = size * nmemb;
if (data->file)
data->file->write((char*)ptr, numBytes);
if (data->str)
*data->str += std::string((char*)ptr, numBytes);
return numBytes;
}
//function to get coin data and perform analysis
int getData()
{
int count = 0;
//begin non terminating loop
while(true)
{
count++;
CURL *curl = curl_easy_init();
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, "http://pubapi.cryptsy.com/api.php?method=singlemarketdata&marketid=155");
std::fstream file("/home/coinz/cryptsy/myfile.txt", std::ios_base::out | std::ios_base::ate);
std::string json;
myData data;
data.file = &file;
data.str = &json;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
/* Perform the request, res will get the return code */
CURLcode res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK)
{
std::cerr << "curl_easy_perform() failed: " << curl_easy_strerror(res) << std::endl;
}
else
{
file << std::endl;
//begin deserialization
Document document;
document.Parse(json.c_str());
assert(document.HasMember("success"));
assert(document["success"].IsString());
//std::cout << "The Last Traded Price is = " << document["lasttradeprice"].GetString() << std::endl;
}
/* always cleanup */
curl_easy_cleanup(curl);
}
//timer for URL request. *ADUJST ME AS DESIRED*
usleep(10000000);
}
return 0;
}
//Le Main
int main(void)
{
getData();
}
Looking at your JSON data, the success member is an int (as it is not surrounded by quote marks).
More info on JSON types can be found here: http://en.wikipedia.org/wiki/JSON#Data_types.2C_syntax_and_example
If the success member was a string, the JSON data would look like this: {"success":"1",...
Therefore when you call assert(document["success"].IsString()) it causes the error, as the success member is not a string, but an int.
Since the success member is an int, change the assertion to test whether it's an int - assert(document["success"].IsInt())
You stated you changed datatypes, but did not state which ones you tried. From the URL you are using, success is being returned as "success":1, which is an int.
Change:
assert(document["success"].IsString());
to:
assert(document["success"].IsInt());