In the project I am working on , I am required to make a GET call and POST call when ever necessary. The "GET" call is used to create a downchannel from the server to the device to monitor for any interesting events that are happen in the cloud server. The "POST" call is used to send the data to the server. The server has a restriction that only a single connection has to be maintained with the server. Since the server supports HTTP2 , I am using "CURLMOPT_MAX_HOST_CONNECTIONS" and by setting it to 1 , I am making CURL to use 1 connection and I am using CURLMOPT_PIPELINING and by setting it to CURLPIPE_MULTIPLEX , I am multiplexing the transfers. The issue is that the code crashes when "curl_multi_perform" is called on the second thread.
PS: This is a part of my original code and might not compile/run straight away.
#include <stdio.h>
#include <string>
#include <sstream>
#include <curl.h>
#include <pthread.h>
#define EVENTS_URL "https://avs-alexa-na.amazon.com/v20160207/events"
#define DOWNCHANNEL_URL "https://avs-alexa-na.amazon.com/v20160207/directives"
CURLM *multi_handle;
CURL *downchannel_handle;
CURL *eventHttp_handle;
size_t processDownchannelResponse(void *ptr, size_t size, size_t nmemb, void *instance)
{
printf(" There is downchannel response.");
}
size_t eventResponse(void *ptr, size_t size, size_t nmemb, void *instance)
{
printf("We got eventResponse");
}
/* The downchannel thread needs to be running always
, if the transfer is done the connection is closed
we need to open a new connection and wait for more events*/
void createDownchannel()
{
int retryCount =0;
std::string downchURL = DOWNCHANNEL_URL;
long response_code;
int runninghandles =0;
downchannel_handle = curl_easy_init();
if(downchannel_handle == NULL){
printf("createDownchannel : Not able to create curl handle");
return ;
}
struct curl_slist *header = NULL;
header = curl_slist_append(header, "Host: avs-alexa-na.amazon.com");
header = curl_slist_append(header, "TOKEN GOES HERE");
curl_easy_setopt(downchannel_handle, CURLOPT_URL, downchURL.c_str());
curl_easy_setopt(downchannel_handle, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(downchannel_handle, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(downchannel_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE);
curl_easy_setopt(downchannel_handle, CURLOPT_WRITEFUNCTION, processDownchannelResponse);
curl_easy_setopt(downchannel_handle, CURLOPT_WRITEDATA, NULL);
curl_easy_setopt(downchannel_handle, CURLOPT_PIPEWAIT, 1L);
if (header)
curl_easy_setopt(downchannel_handle, CURLOPT_HTTPHEADER, header);
if(multi_handle)
/* add the individual easy handle */
curl_multi_add_handle(multi_handle, downchannel_handle);
mbexitdownchannelthread = 1;
do{
CURLMcode mc;
int numfds;
mc = curl_multi_perform(multi_handle, &runninghandles);
if(mc != CURLM_OK) {
fprintf(stderr, "curl_multi failed, code %d.n", mc);
printf(“a<---curl_multi failed, code %d.n", mc);
break;
}
printf("a<--downchannel while");
// downchannel timeout can be higher 10 seconds
mc = curl_multi_wait(multi_handle, NULL, 0, 10000, &numfds);
if(mc != CURLM_OK) {
fprintf(stderr, "curl_multi failed, code %d.n", mc);
printf("a<---curl_multi failed, code %d.n", mc);
break;
}
}while(!mbexitdownchannelthread);
curl_multi_remove_handle(multi_handle, downchannel_handle);
curl_easy_cleanup(downchannel_handle);
if (header)
curl_slist_free_all(header);
}
bool getTransferStatus(CURLM *multiHandleInstance,CURL *currentHandle,std::string& contentType,int* http_status_code)
{
//check the status of transfer CURLcode return_code=0;
int msgs_left=0;
CURLMsg *msg=NULL;
CURL *eh=NULL;
CURLcode return_code;
bool msgTrasferDone = false;
while ((msg = curl_multi_info_read(multiHandleInstance, &msgs_left)))
{
if (msg->msg == CURLMSG_DONE) {
eh = msg->easy_handle;
if(currentHandle == eh)
{
msgTrasferDone = true;
return_code = msg->data.result;
if(return_code!=CURLE_OK)
{
//fprintf(stderr, "CURL error code: %d\n", msg->data.result);
printf("a<--return_code!=CURLE_OK CURL error code: %d\n", msg->data.result);
continue;
}
// Get HTTP status code
*http_status_code=0;
char* ch= NULL;
curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, http_status_code);
printf("a<--CURLINFO_RESPONSE_CODE=%d",*http_status_code);
curl_easy_getinfo(eh, CURLINFO_CONTENT_TYPE, &ch);
contentType.clear();
if(ch!=NULL)
contentType.append(ch);
}
}
}
return msgTrasferDone;
}
int eventThread()
{
int still_running = 0;
std::string contentType;
int responseCode = 0;
int numfds = 0;
std::string postUrl;
postUrl.assign(EVENTS_URL);
// init the curl session
CURL *eventHttp_handle;
CURLcode res;
eventHttp_handle = curl_easy_init();
//assign speech buffer pointer for read callbacks
curl_easy_setopt(eventHttp_handle, CURLOPT_URL, postUrl.c_str());
curl_easy_setopt(eventHttp_handle, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(eventHttp_handle, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(eventHttp_handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE);
/* wait for pipe connection to confirm */
curl_easy_setopt(eventHttp_handle, CURLOPT_PIPEWAIT, 1L);
curl_easy_setopt(eventHttp_handle, CURLOPT_WRITEFUNCTION, eventResponse);
curl_easy_setopt(eventHttp_handle, CURLOPT_WRITEDATA, NULL);
struct curl_slist *header = NULL;
header = curl_slist_append(header, "Host: avs-alexa-na.amazon.com");
header = curl_slist_append(header, "TOKEN GOES HERE");
if (header)
curl_easy_setopt(eventHttp_handle, CURLOPT_HTTPHEADER, header);
if (multi_handle)
curl_multi_add_handle(multi_handle, eventHttp_handle);
do
{
{
CURLMcode mc;
int numfds;
mc = curl_multi_perform(multi_handle, &still_running);
if(mc != CURLM_OK) {
fprintf(stderr, "curl_multi failed, code %d.n", mc);
printf("a<---curl_multi failed, code %d.n", mc);
break;
}
mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
if(mc != CURLM_OK) {
fprintf(stderr, "curl_multi failed, code %d.n", mc);
printf("a<---curl_multi failed, code %d.n", mc);
break;
}
}
}while(!getTransferStatus(multi_handle,eventHttp_handle,contentType,&responseCode));
if (header)
curl_slist_free_all(header);
if (formpost)
{
curl_formfree(formpost);
formpost = NULL;
}
printf("a<-- CURL HTTP RESP CODE =%d",responseCode);
curl_multi_remove_handle(multi_handle, eventHttp_handle);
curl_easy_cleanup(eventHttp_handle);
return 0;
}
int main ()
{
multi_handle = curl_multi_init();
curl_multi_setopt(multi_handle, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, 1L);
// create a downchannel thread
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
int err = pthread_create(&mPostThread, &attr, downchannelThread,NULL);
pthread_attr_destroy(&attr);
// i am doing some work here //
sleep(3);
// create a event thread
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
int err = pthread_create(&mPostThread, &attr, eventThread,NULL);
pthread_attr_destroy(&attr);
}
Related
I am very new to libcurl. So any help is appreciated.
I am trying to download multiple files from CDN. The total size is around 600mb.
This is the piece of code where I am setting CURL handles(Inside a method named "addTransfers()"). (cm is of type CURLM*)
CURL *ceh = curl_easy_init();
curl_easy_setopt(ceh, CURLOPT_URL, reqURL.c_str());
curl_easy_setopt(ceh, CURLOPT_USERPWD, usrpwd);
curl_easy_setopt(ceh, CURLOPT_WRITEDATA, file);
curl_easy_setopt(ceh, CURLOPT_PRIVATE, file);
curl_easy_setopt(ceh, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(ceh, CURLOPT_FAILONERROR, 1L);
curl_easy_setopt(ceh, CURLOPT_CONNECTTIMEOUT, 10L);
curl_easy_setopt(ceh, CURLOPT_TIMEOUT, 150L);
curl_multi_add_handle(cm, ceh);
This is the code where I am performing the curl multi process:
bool do_multiple_downloads()
{
CURLM *cm;
CURLMsg *msg;
unsigned int transfers = 0;
int pending = 1;
int msg_pending = -1;
curl_global_init(CURL_GLOBAL_ALL);
cm = curl_multi_init();
curl_multi_setopt(cm, CURLMOPT_MAXCONNECTS, maxConnects);
for(transfers = 0; transfers < (int)maxConnects; transfers++)
{
if(!addTransfers(cm, transfers))
{
//Error
return false;
}
}
int i = 0;
do
{
/* code */
CURLMcode cPerformCode = curl_multi_perform(cm, &pending);
if(cPerformCode != CURLM_OK)
{
fprintf(stderr, "Error --> (%s) : Pending --> (%d)", curl_multi_strerror(cPerformCode), pending);
return false;
}
while((msg = curl_multi_info_read(cm, &msg_pending)))
{
if(msg != NULL)
{
if(msg->msg == CURLMSG_DONE)
{
FILE* writtenFile;
char* fname;
CURL *e = msg->easy_handle;
curl_easy_getinfo(e, CURLINFO_PRIVATE, &writtenFile);
fprintf(stderr, "R: %d - %s (%d)\n",
msg->data.result, curl_easy_strerror(msg->data.result), i++);
curl_multi_remove_handle(cm, e);
fclose(writtenFile);
curl_easy_cleanup(e);
}
else
{
fprintf(stderr, "E: CURLMsg (%d) : (%d)\n", msg->msg, i++);
}
if(transfers < totalSize)
{
if(!addTransfers(cm, transfers++))
{
//Error
fprintf(stderr, "Additional transfers addition\n");
return false;
}
}
}
}
if(pending)
{
curl_multi_wait(cm, NULL, 0, 1000, NULL);
}
} while (pending || (transfers < totalSize));
curl_multi_cleanup(cm);
curl_global_cleanup();
return true;
}
These are the scenarios:
Sometimes it works fine and all files are downloaded - No problem
Sometimes the downloads are completed, but the program is still inside the do-while loop(pending variable never becomes zero)
If I disconnect from the internet when the process is happening, the program stays inside the do-while loop. When I reconnect to the internet, Scenario 2 occurs.
Am I missing something here?? I might have done something silly here. Not sure though.
Thanks in advance.
I am currently using the multi interface API of the libcurl library to send multiple HTTP requests without blocking. My current problem is that I can't send a HTTP request.
For some reason maxfd is always -1 which result in rc always being 0. So when the code get to the switch statement it always print "timeout" because rc is always 0.
I never used the mulit interface before but I used the easy interface a couple of time. Much of the code is very similar to this.
ReqestHandler.cpp
void RequestHandler::add(lua_State* lua, const std::string& url, const std::string& data) {
Request request;
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "RoBot/ Version 1");
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L);
curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 0);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &request.response);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &request.headers);
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &request.code);
if (!data.empty()) {
curl_easy_setopt(curl, CURLOPT_POST, true);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str());
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.size());
}
request.curl = curl;
request.lua = lua;
curl_multi_add_handle(_curlm, curl);
_requests.push_back(request);
}
void RequestHandler::tick() {
if (!_requests.empty()) {
curl_multi_perform(_curlm, &_isRunning);
do {
struct timeval timeout;
int rc;
CURLMcode mc;
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd = -1;
long curl_timeo = -1;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
timeout.tv_sec = 1;
timeout.tv_usec = 0;
curl_multi_timeout(_curlm, &curl_timeo);
if (curl_timeo >= 0) {
timeout.tv_sec = curl_timeo / 1000;
if (timeout.tv_sec > 1) {
timeout.tv_sec = 1;
} else {
timeout.tv_usec = (curl_timeo % 1000) * 1000;
}
}
mc = curl_multi_fdset(_curlm, &fdread, &fdwrite, &fdexcep, &maxfd);
if (mc != CURLM_OK) {
throw new Exception("MC error code: " + mc);
break;
}
if (maxfd == -1) {
Sleep(1000);
rc = 0;
} else {
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
}
switch (rc) {
case -1:
/* select error */
std::cout << "Error" << std::endl;
break;
case 0:
/* timeout */
std::cout << "Timeout" << std::endl;
break;
default:
/* action */
std::cout << "Action" << std::endl;
curl_multi_perform(_curlm, &_isRunning);
break;
}
} while (_isRunning);
while ((_msg = curl_multi_info_read(_curlm, &_msgsLeft))) {
std::cout << "RAN";
if (_msg->msg == CURLMSG_DONE) {
for (unsigned int i = 0; i < _requests.size(); i++) {
Request request = _requests[i];
if (_msg->easy_handle == request.curl) {
std::cout << request.response << std::endl;
break;
}
}
}
}
curl_multi_cleanup(_curlm);
for (unsigned int i = 0; i < _requests.size(); i++) {
curl_easy_cleanup(_requests[i].curl);
_requests.erase(_requests.begin() + i);
}
}
}
RequestHandler.h
#pragma once
#include <string>
#include <vector>
#include "curl\curl.h"
#include "Lua\lua.hpp"
struct Request {
std::string response;
std::string headers;
int code;
CURL* curl;
lua_State* lua;
};
class RequestHandler {
public:
RequestHandler();
~RequestHandler();
static RequestHandler* getInstance();
void add(lua_State* lua, const std::string& url, const std::string& data);
void tick();
private:
static RequestHandler* INSTANCE;
std::vector<Request> _requests;
CURLM* _curlm;
CURLMsg* _msg;
int _isRunning;
int _msgsLeft;
};
Update
As suggested in the comment I used CURLOPT_STDERR and CURLOPT_VERBOSE and nothing printed to the output file.
I added this code within the RequestHandler:add(); function.
#ifdef _DEBUG
FILE* file = fopen("../Logs/Verbose.log", "wb");
curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
curl_easy_setopt(curl, CURLOPT_STDERR, file);
#endif
The file does get created but nothing get written to the file.
when i use hmac and curl then the programe coredump. And the log run there
"Oct 17 17:58:37 info [6382]: 6385 httpsPost before apped headers"
I got the infomation in gdb
Program terminated with signal 11, Segmentation fault.
#0 0x0819a8e7 in EVP_MD_CTX_md ()
(gdb) where
#0 0x0819a8e7 in EVP_MD_CTX_md ()
Cannot access memory at address 0x4004
these information is no help to me,is anyone know about it?
but when I delete the code:
HMAC_CTX ctx;
HMAC_CTX_init(&ctx);
HMAC_Init_ex(&ctx, AUTH_KEY, strlen(AUTH_KEY), EVP_sha256(), NULL);
HMAC_Update(&ctx, (unsigned char*)&szJsonData, strlen(szJsonData));
HMAC_Final(&ctx, result, &len);
HMAC_CTX_cleanup(&ctx);
it run well,I don't know why
I got these information :
successfully set certificate verify locations:
CAfile: none
CApath: /etc/ssl/certs
void postHttps(uint32_t uid,std::map& headResult,std::map& bodyResult)
{
char szJsonData[1024];
memset(szJsonData, 0, sizeof(szJsonData));
std::ostringstream requetContent;
requetContent<<"{\"cmuid\":\""<<uid<<"\",\"s\":2}";
strcpy(szJsonData, requetContent.str().c_str());
char bodyBuff[1024];
memset(bodyBuff, 0, sizeof(bodyBuff));
char headBuff[1024];
memset(headBuff, 0, sizeof(headBuff));
uint32_t start = sox::env::now;
uint32_t ssend = sox::env::now;
unsigned char* result;
char sign[1024] = {0};
unsigned int len = 256;
result = (unsigned char*)malloc(sizeof(char) * len);
log(Info,"postHttps accpet");
HMAC_CTX ctx;
HMAC_CTX_init(&ctx);
HMAC_Init_ex(&ctx, AUTH_KEY, strlen(AUTH_KEY), EVP_sha256(), NULL);
HMAC_Update(&ctx, (unsigned char*)&szJsonData, strlen(szJsonData));
HMAC_Final(&ctx, result, &len);
HMAC_CTX_cleanup(&ctx);
for (unsigned int i = 0; i != len; i++){
int length = strlen(sign);
char *s = sign;
sprintf(s + length, "%02x", (unsigned int)result[i]);
}
free(result);
string auth_head = "Authorization:";
auth_head += UrlEncode(sign).c_str();
log(Info,"postHttps requetContent %s auth_head %s size %u len %u", requetContent.str().c_str(),auth_head.c_str(),requetContent.str().size(),strlen(szJsonData));
try
{
CURL *pCurl = NULL;
CURLcode res;
pCurl = curl_easy_init();
if (NULL != pCurl)
{
curl_easy_setopt(pCurl, CURLOPT_TIMEOUT, 2);
curl_easy_setopt(pCurl, CURLOPT_URL, "https://www.xxxx.com/id");
curl_easy_setopt(pCurl, CURLOPT_POST, 1L);
curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYPEER, false);
curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYHOST, false);
curl_easy_setopt(pCurl, CURLOPT_CONNECTTIMEOUT, 2);
//curl_easy_setopt(pCurl, CURLOPT_VERBOSE, 1);
//curl_easy_setopt(pCurl, CURLOPT_UPLOAD, 1);
curl_easy_setopt(pCurl, CURLOPT_NOSIGNAL, 1);
struct curl_slist *headers = NULL;
log(Info,"httpsPost before apped headers");
headers = curl_slist_append(headers, auth_head.c_str());
headers = curl_slist_append(headers, "Accept: text/plain");
headers = curl_slist_append(headers, "charset:utf-8");
headers = curl_slist_append(headers, "Content-Type: application/json");
curl_easy_setopt(pCurl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(pCurl, CURLOPT_FORBID_REUSE, 1);
curl_easy_setopt(pCurl, CURLOPT_WRITEFUNCTION, on_writebuff);
curl_easy_setopt(pCurl, CURLOPT_WRITEDATA, bodyBuff);
curl_easy_setopt(pCurl, CURLOPT_HEADERFUNCTION, header_handler);
curl_easy_setopt(pCurl, CURLOPT_WRITEHEADER, headBuff);
curl_easy_setopt(pCurl,CURLOPT_POSTFIELDS, requetContent.str().c_str());
curl_easy_setopt(pCurl, CURLOPT_POSTFIELDSIZE,requetContent.str().size());
int HTTP_flag = 0;
curl_easy_getinfo(pCurl, CURLINFO_RESPONSE_CODE, &HTTP_flag);
res = curl_easy_perform(pCurl);
log(Info,"httpsPost after curl_easy_perform");
curl_slist_free_all(headers);
curl_easy_cleanup(pCurl);
if (res == CURLE_OK)
{
std::string httpRes = headBuff;
HttpParser::ParseResponseHead(httpRes,headResult);
std::map<std::string,std::string>::iterator it = headResult.find("HTTP");
if(it != headResult.end())
{
std::string& value = it->second;
if(value != "200 OK")
{
log(Error,"postHttps HTTP %s", value.c_str());
}
else
{
json_char* json = (json_char*)bodyBuff;
json_value* value = json_parse(json,strlen(bodyBuff));
process_value(value, 0,bodyResult);
json_value_free(value);
log(Info,"httpsPost headMap %d bodyMap %d", headResult.size(),bodyResult.size());
}
}
}
else
{
log(Info,"curl_easy_perform() failed:%s", curl_easy_strerror(res));
}
}
}
catch (std::exception &ex)
{
log(Info,"curl exception %s", ex.what());
}
log(Info,"httpPost uid %u elapse %u", uid,ssend-start);
}
use ssl,you should init some thinds. like below
void Callbackfee::init_locks(void)
{
int i;
if(m_lockarray == NULL)
{
m_lockarray = (pthread_mutex_t *)OPENSSL_malloc(CRYPTO_num_locks() *
sizeof(pthread_mutex_t));
}
if(m_lockarray != NULL)
{
for (i = 0; i < CRYPTO_num_locks(); ++i) {
pthread_mutex_init(&(m_lockarray[i]), NULL);
}
CRYPTO_set_id_callback(&thread_id);
CRYPTO_set_locking_callback(&lock_callback);
}
}
I'm trying to upload a file to an http server. I'm getting the 200 OK from the server, but the code below is only transmitting 4 bytes.
size_t myclass::read_callback(void *ptr, size_t size, size_t nmemb, void *userp)
{
handler->read(buffer, buffer_size); // buffer_size is 100000
size_t res = handler->gcount();
if( res == 0 )
return 0;
ptr = buffer; // buffer is array of char, defined in myclass
size = res;
nmemb = sizeof(char);
return 1;
}
void myclass::upload_function(const std::string& url)
{
CURL *curl;
CURLcode res;
std::ifstream if_file;
if_file.open("/path_to_file", std::ios::binary);
handler = &if_file; // handler is defined in myclass
/* In windows, this will init the winsock stuff */
res = curl_global_init(CURL_GLOBAL_DEFAULT);
/* Check for errors */
if(res != CURLE_OK) {
// failure
return;
}
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "hostname");
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, myclass::read_callback);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
struct curl_slist *chunk = NULL;
chunk = curl_slist_append(chunk, "Transfer-Encoding: chunked");
chunk = curl_slist_append(chunk, "Content-Type: application/x-mpegURL");
res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK) {
// failed
}
else
{
double speed_upload, total_time;
curl_easy_getinfo(curl, CURLINFO_SPEED_UPLOAD, &speed_upload);
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time);
fprintf(stderr, "Speed: %0.3f b/sec during %.3f seconds\n",
speed_upload, total_time);
}
/* always cleanup */
curl_easy_cleanup(curl);
}
curl_global_cleanup();
if_file.close();
}
The callback doesn't seem to copy data to the buffer. It just assigns the local pointer, quite without any effect.
The callback looks like to be a C++ method that can't be used like that as a callback in a C API that doesn't know about C++ objects...
I am working on a C++ program, which uses cURL lib. When I try to connect to URL, that is incorrect or the page is down I get following error :
getaddrinfo(3) failed for www.wp.pl#:80
Couldn't resolve host 'www.wp.pl#'
And the program terminates. And that is the problem, because I it to go futher and connect to other pages. I just won't parse and analyze the broken page I don't want to end my program after come across such one.
I posted similar topic, but it contained only snippets of code so it was diffcult for you to say what is wrong, now I will post whole code.
int PageHandler::getCleanAndRepairedPage(string pageURL, string& outDoc)
throw (exception) {
if (find(visitedPages.begin(), visitedPages.end(), pageURL)
!= visitedPages.end()) {
// url has already been visited
outDoc = "";
return VISITED_PAGE_ERROR;
} else { // new url
visitedPages.push_back(pageURL);
char *charURL;
charURL = const_cast<char*> (pageURL.c_str());
CURL *curl;
char curl_errbuf[CURL_ERROR_SIZE];
TidyBuffer output = { 0 };
TidyBuffer errbuf = { 0 };
TidyBuffer docbuf = { 0 };
TidyDoc tdoc = tidyCreate(); // Initialize "document"
tidyOptSetBool(tdoc, TidyForceOutput, yes);
tidyOptSetInt(tdoc, TidyWrapLen, 4096);
tidyBufInit(&docbuf);
int rc = -1;
Bool ok;
curl = curl_easy_init();
curl_easy_setopt( curl, CURLOPT_URL, charURL );
curl_easy_setopt( curl, CURLOPT_ERRORBUFFER, curl_errbuf );
curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_cb );
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt( curl , CURLOPT_WRITEDATA, &docbuf );
int err;
err = curl_easy_perform(curl);
cout << "curl_easy_perfom return value = " << err << endl;
if (!err) {
ok = tidyOptSetBool(tdoc, TidyXhtmlOut, yes); // Convert to XHTML
if (ok) {
rc = tidySetErrorBuffer(tdoc, &errbuf); // Capture diagnostics
if (rc >= 0) {
rc = tidyParseBuffer(tdoc, &docbuf); // parse the buffer
}
if (rc >= 0) {
rc = tidyCleanAndRepair(tdoc); // Tidy it up!
}
if (rc >= 0) {
rc = tidyRunDiagnostics(tdoc); // Kvetch
}
if (rc > 1) { // If error, force output.
rc = (tidyOptSetBool(tdoc, TidyForceOutput, yes) ? rc : -1);
}
if (rc >= 0) {
rc = tidySaveBuffer(tdoc, &output); // Pretty Print
}
}
if (rc >= 0) {
if (rc > 0) {
//printf("\nDiagnostics:\n\n%s", errbuf.bp);
}
} else {
printf("A severe error (%d) occurred.\n", rc);
}
} else {
printf("%s\n", curl_errbuf);
}
if (err == NO_ERROR) {
string tmp(reinterpret_cast<char const*> (output.bp));
outDoc = tmp;
} else {
outDoc = "";
}
curl_easy_cleanup(curl);
tidyBufFree(&docbuf);
tidyBufFree(&output);
tidyBufFree(&errbuf);
tidyRelease(tdoc);
return err; // err == 0 <=> everything ok!
}
}
And the console output :
EDIT : I forgot to say what is going on on console output. I connect to many pages and the invalid URL is just one of them. At the beginning you can see a succesful connection and cURL messages about it, the result of curl_easy_perform equals 0, which indicate that everything went good. Next message is about connection to the invalid URL, as u can see return value of curl_easy_perform is 6, which is not good.
"www.wp.pl#" is not a legal URL and it causes libcurl to (mistakenly) use the entire string as a host name, and you don't have any such and the operation fails.