thread-safety proxy with libcurl - c++

#pragma once
#ifndef __CURL_CURL_H
#include "curl.h"
#endif
#ifndef __CURL_EASY_H
#include "easy.h"
#endif
#include <stdint.h>
#include <memory>
#include <string>
namespace CommUnit
{
enum ERR_PROXY
{
ERR_CURL_INIT_FAILED = 0xA0,
ERR_SET_PROXY_FAILED = 0xA1,
};
class MyProxy
{
public:
static MyProxy & GetInstance() //Meyers' Singlton
{
static MyProxy ProxySigleton;
return ProxySigleton;
}
public:
/*
* #bref:Get request
* #param[in] sUrl:Access URL
* #param[in] sProxyIp:Proxy IP
* #param[in] uProxyPort:Proxy Port
* #param[in] uTimeOut:Time out
* #param[in] isSSL:HTTPS true,else false
* #param[out] sRetContent:Return the URL content
*/
uint32_t Get(const std::string &sUrl,
const std::string& sProxyIp,
uint32_t uProxyPort,
uint32_t uTimeOut,
bool isSSL,
std::string &sRetContent);
private:
MyProxy(); //Constructor hidden
MyProxy(MyProxy const &); //Copy-Constructor hidden
MyProxy & operator= (MyProxy const &); //Assign operator hidden
~MyProxy(); //Destructor hidden
inline void _setCurlopt(CURL *pCurl,
const std::string &sUrl,
std::string &sWriterData,
const uint32_t uTimeOut,
bool isSSL);
//Callback function, write data to writerData
static int Writer(char *data,
uint32_t size,
uint32_t nmemb,
std::string *writerData);
private:
std::string m_sErrMsg;
static char s_ErrBuffer[CURL_ERROR_SIZE];
static const uint32_t m_MAXBUF = 2 * 1024 * 1024 - 128;
};
}
//////////////////////////////////////////////////////////////////////////
#include <iostream>
#include <string>
#include "MyProxy.h"
#include "Log.h"
#include <curl.h>
using namespace CommUnit;
char MyProxy::s_ErrBuffer[CURL_ERROR_SIZE] = { 0 };
MyProxy::MyProxy(void)
{
CURLcode oCURLcode = curl_global_init(CURL_GLOBAL_ALL);
if (oCURLcode != CURLE_OK)
{
Log("ERR: %s curl_init failed!", __func__);
}
}
MyProxy::~MyProxy(void)
{
curl_global_cleanup();
}
uint32_t MyProxy::Get(const std::string &sUrl,
const std::string& sProxyIp,
uint32_t uProxyPort,
uint32_t uTimeOut,
bool isSSL,
std::string &sRetContent)
{
sRetContent.clear();
CURL *pCurl = curl_easy_init();
CURLcode oCURLcode;
if (nullptr == pCurl)
{
Log("ERR: %s curl_easy_init failed!", __func__);
return ERR_CURL_INIT_FAILED;
}
_setCurlopt(pCurl, sUrl, sRetContent, uTimeOut, isSSL);
if (0 == sProxyIp.length()|| 0 == uProxyPort)
{
Log("ERR: %s SetProxy: ProxyIp [%s], ProxyPort[%u] failed",__func__, sProxyIp.c_str(), uProxyPort);
return ERR_SET_PROXY_FAILED;
}
Log("INFO: %s SetProxy: ProxyIp [%s], ProxyPort[%u] failed", __func__, sProxyIp.c_str(), uProxyPort);
curl_easy_setopt(pCurl, CURLOPT_PROXY, sProxyIp.c_str());
curl_easy_setopt(pCurl, CURLOPT_PROXYPORT, uProxyPort);
int iTimes = 0;
while (true)
{
oCURLcode = curl_easy_perform(pCurl);
if (oCURLcode != CURLE_OK && ++iTimes < 3)
usleep(5);
else
break;
}
if (oCURLcode != CURLE_OK)
{
Log("ERR: %s curl_easy_perform failed!", __func__);
}
curl_easy_cleanup(pCurl);
return oCURLcode;
}
void MyProxy::_setCurlopt(CURL *pCurl,
const std::string &sUrl,
std::string &sWriterData,
const uint32_t uTimeOut,
bool isSSL)
{
curl_easy_setopt(pCurl, CURLOPT_ERRORBUFFER, s_ErrBuffer);
curl_easy_setopt(pCurl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(pCurl, CURLOPT_URL, sUrl.c_str());
curl_easy_setopt(pCurl, CURLOPT_TIMEOUT, uTimeOut);
Log("INFO: %s Set Url:[%s],TimeOut:[%d]", __func__, sUrl.c_str(), uTimeOut);
curl_easy_setopt(pCurl, CURLOPT_WRITEFUNCTION, MyProxy::Writer);
curl_easy_setopt(pCurl, CURLOPT_WRITEDATA, &sWriterData);
//Skip peer and hostname verification
if (isSSL)
{
curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYHOST, 0L);
}
}
int MyProxy::Writer(char *data,
uint32_t size,
uint32_t nmemb,
std::string *writerData)
{
if (writerData == nullptr)
{
Log("ERR: %s writerData is null!", __func__);
return 0;
}
int len = size * nmemb;
if ((writerData->size() + len) > m_MAXBUF)
{
Log("ERR: %s writerData size over MAXBUF!", __func__);
return 0;
}
writerData->append(data, len);
return len;
}
I want to realize a proxy with libcurl, which can get the content of given url(https). Morever, it need to be thread-safety.
But when I created 200 threads with pthreads to test my code, an segment fault occured sometimes.
How can I solve this problem?
Is there a relation with the sRetContent(std::string)?
Thanks!
Errmsg:
double free or corruption (!prev): 0x0ac72840 ***
Segmentation fault

My understanding is that libcurl is not thread-safe if you are using https (and it looks like you are) due to the fact that it is using underlying ssl libraries. See libcurl documentation and OpenSSL documentation for more info.
If your libcurl was compiled with OpenSSL for example then you have to initialize a few callback functions or you could run into issues. This is the sort of thing you need to do (compiles on Windows):
#include <curl/curl.h>
#include <openssl/crypto.h>
void win32_locking_callback(int mode, int type, const char *file, int line)
{
if (mode & CRYPTO_LOCK)
{
WaitForSingleObject(lock_cs[type],INFINITE);
}
else
{
ReleaseMutex(lock_cs[type]);
}
}
void thread_setup(void)
{
int i;
lock_cs=(HANDLE*)OPENSSL_malloc(CRYPTO_num_locks() * sizeof(HANDLE));
for (i=0; i<CRYPTO_num_locks(); i++)
{
lock_cs[i]=CreateMutex(NULL,FALSE,NULL);
}
CRYPTO_set_locking_callback((void (*)(int,int,const char *,int))win32_locking_callback);
}
void thread_cleanup(void)
{
int i;
CRYPTO_set_locking_callback(NULL);
for (i=0; i<CRYPTO_num_locks(); i++)
CloseHandle(lock_cs[i]);
OPENSSL_free(lock_cs);
}
I always call thread_setup() after my call to curl_global_init(CURL_GLOBAL_ALL)
and then thread_cleanup() right before my call to curl_global_cleanup().
I use this sort of code with libcurl often in load test scenarios and have never run into any issues. If you continue to run into problems, it is not libcurl, but something not being done properly in your code.

libcurl is thread-safe as long as you play by the rules

Related

write access violation with c++ libcurl [closed]

Closed. This question is not reproducible or was caused by typos. It is not currently accepting answers.
This question was caused by a typo or a problem that can no longer be reproduced. While similar questions may be on-topic here, this one was resolved in a way less likely to help future readers.
Closed last month.
Improve this question
I'm trying to create c++ code that download data from some URLs, but it's throwing a write access violation:
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#ifndef WIN32
#endif
#include <curl/curl.h>
#include <string>
static const char* urls[] = {
"http://www.example.com",
"http://www.example1.com",
};
#define MAX_PARALLEL 10 /* number of simultaneous transfers */
#define NUM_URLS sizeof(urls)/sizeof(char *)
static size_t write_cb(void* ptr, size_t size, size_t nmemb, void* buffer)
{
((std::string*)buffer)->append((char*)ptr, nmemb);
return nmemb;
}
static void add_transfer(CURLM* cm, int i, int* left)
{
CURL* eh = curl_easy_init();
curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, write_cb);
curl_easy_setopt(eh, CURLOPT_URL, urls[i]);
curl_easy_setopt(eh, CURLOPT_PRIVATE, urls[i]);
curl_easy_setopt(eh, CURLOPT_WRITEDATA, &write_cb);
curl_easy_setopt(eh, CURLOPT_VERBOSE, 1L);
curl_multi_add_handle(cm, eh);
(*left)++;
}
int main(void)
{
CURLM* cm;
unsigned int transfers = 0;
int msgs_left = -1;
int left = 0;
curl_global_init(CURL_GLOBAL_ALL);
cm = curl_multi_init();
/* Limit the amount of simultaneous connections curl should allow: */
curl_multi_setopt(cm, CURLMOPT_MAXCONNECTS, (long)MAX_PARALLEL);
for (transfers = 0; transfers < MAX_PARALLEL && transfers < NUM_URLS;
transfers++)
add_transfer(cm, transfers, &left);
do {
int still_alive = 1;
curl_multi_perform(cm, &still_alive);
CURLMsg* msg;
int queued;
CURLMcode mc = curl_multi_perform(cm, &still_alive);
if (cm)
/* wait for activity, timeout or "nothing" */
mc = curl_multi_poll(cm, NULL, 0, 1000, NULL);
if (mc)
break;
do {
msg = curl_multi_info_read(cm, &queued);
if (msg) {
if (msg->msg == CURLMSG_DONE) {
/* a transfer ended */
fprintf(stderr, "Transfer completed\n");
}
}
} while (msg);
if (left)
curl_multi_wait(cm, NULL, 0, 1000, NULL);
} while (left);
curl_multi_cleanup(cm);
curl_global_cleanup();
return EXIT_SUCCESS;
}
Its crashing on the line:
_Mypair._Myval2._Mysize = _Old_size + _Count;
The full error message is:
Exception thrown: write access violation.
this was 0x7FF7941D39D0.
How can I make this code download each Url data without any error?
In your write_cb() callback, you are expecting the buffer parameter to point at a std::string object, but in add_transfer() you are setting CURLOPT_WRITEDATA to point at write_cb itself rather than at a std::string object.
Try something more like this instead:
struct url_info {
const char* url;
std::string data;
};
static url_info urls[] = {
{"http://www.example.com", ""},
{"http://www.example1.com", ""}
};
static const int NUM_URLS = sizeof(urls)/sizeof(urls[0]);
static size_t write_cb(void* ptr, size_t size, size_t nmemb, void* buffer)
{
size_t result = size * nmemb;
static_cast<std::string*>(buffer)->append(static_cast<char*>(ptr), result);
return result;
}
static void add_transfer(CURLM* cm, int i, int* left)
{
...
curl_easy_setopt(eh, CURLOPT_URL, urls[i].url);
curl_easy_setopt(eh, CURLOPT_WRITEDATA, &urls[i].data);
curl_easy_setopt(eh, CURLOPT_PRIVATE, &urls[i]);
...
}
int main(void)
{
...
while ((msg = curl_multi_info_read(cm, &queued)) != NULL) {
if (msg->msg == CURLMSG_DONE) {
/* a transfer ended */
url_info *info;
curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, reinterpret_cast<char**>(&info));
// use info->url and info->data as needed...
std::cerr << "Transfer completed from " << info->url << ", bytes received: " << info->data.size() << "\n";
}
}
...
}

Same code works/gives error in different projects

I am using visual studio 2019.
When I try to run this from a new project it works totally fine. Sends the email and doesn't give any error.
But when I try to run this from my server code it gives this error:
To be more precise:
Project1/DEBUG works fine.
Project1/Release works fine.
Project2/DEBUG works fine.
Project2/Release gives runtime error.
"Exception thrown: read access violation.
this was 0xF. occurred"
And compiler is redirected to "xstring" code page, line 2873.
part from xstring
basic_string& assign(_In_reads_(_Count) const _Elem* const _Ptr, _CRT_GUARDOVERFLOW const size_type _Count) {
// assign [_Ptr, _Ptr + _Count)
if (_Count <= _Mypair._Myval2._Myres) { // LINE 2873
_Elem* const _Old_ptr = _Mypair._Myval2._Myptr();
_Mypair._Myval2._Mysize = _Count;
_Traits::move(_Old_ptr, _Ptr, _Count);
_Traits::assign(_Old_ptr[_Count], _Elem());
return *this;
}
I've tried with totally same code (with copy paste) but still no use.
Project properties are the same. I've checked it several times for any change in properties but there is not even a single difference.
email.h
#pragma warning(disable : 4996) //_CRT_SECURE_NO_WARNINGS
#ifndef EMAIL_H
#define EMAIL_H
#include <vector>
#include <string>
#include "curl/curl.h"
// Enable one of the below depending on your operating system
#define WINDOWS_H
//#define LINUX_H
class Email
{
public:
// default constructor
Email();
// sets who the email is going to
void setTo(const std::string to);
// sets who the email came from
void setFrom(const std::string from);
// sets the cc
void setCc(const std::string to);
// sets the subject of the email
void setSubject(const std::string subject);
// set the body of the email
void setBody(const std::string body);
// sets the smtp username
void setSMTP_username(const std::string username);
// sets the smtp password
void setSMTP_password(const std::string password);
// sets the SMTP HOST
void setSMTP_host(const std::string host);
// adds a binary attachment to the email
void addAttachment(const std::string filepath);
// removes an attachment from the email (Not implemented yet)
void removeAttachment(const std::string filepath);
// removes all attachments
void removeAllAttachments();
// contructs the final email
void constructEmail();
// clears the contents of the email
void clearEmailContents();
// void send email
int send() const; // returns a CURL error code
// dumps the email contents for debugging
void dump() const;
private:
// smtp information
std::string smtp_user;
std::string smtp_password;
std::string smtp_host;
// email data
std::string to;
std::string from;
std::string cc;
std::string subject;
std::string body;
// vector which stores the email data
std::vector<std::string> email_contents;
std::vector<std::string> attachments;
// length of the above vector
int numberOfLines;
};
#endif
email.cpp
#include <iostream>
#include <ctime>
#include <string.h>
// #include "stdafx.h"
#include "email.h"
#define MAX_LEN 255 // this must be divisible by 3 otherwise the SMTP server won't be able to decode the attachment properly
#define ENCODED_LEN 341
// offsets into the email template
#define BOUNDARY 5
#define END_LINE 8
#define ATTACH_TYPE 10
#define ATTACH_TRANSFER 11
#define ATTACH_DEPOSITION 12
#define END_OF_TRANSMISSION_BOUNDARY 15
#define END_OF_TRANSMISSION 16
#ifdef WINDOWS_H
#define DIR_CHAR '\\'
#else
#define DIR_CHAR '/'
#endif
using namespace std;
// base64 encoding functions
void base64_encode(char* input_buf, char* output_buf, size_t input_size);
void encodeblock(char* in, char* out, int len);
// callback function
static size_t payload_source(void* ptr, size_t size, size_t nmemb, void* userp);
// base64 encoding table
const char b64_table[65] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
// global vector this is TEMP
vector<string> global_vec;
// struct used by the CURL callback
struct upload_status {
int lines_read;
};
static const char* email_template[] = {
/* i = 0 */
"User-Agent: DXF Viewer agent\r\n",
"MIME-Version: 1.0\r\n",
"Content-Type: multipart/mixed;\r\n",
" boundary=\"------------030203080101020302070708\"\r\n",
"\r\nThis is a multi-part message in MIME format.\r\n",
"--------------030203080101020302070708\r\n", // use this type of boundary for subsequent attachments
"Content-Type: text/plain; charset=utf-8; format=flowed\r\n",
"Content-Transfer-Encoding: 7bit\r\n",
/* i = 8 (BODY) */
"\r\n", /* empty line to divide headers from body, see RFC5322 */
"--------------030203080101020302070708\r\n", /* i = 9 */
/* Template for binary attachments i = 10 */
"Content-Type: application/octet-stream\r\n",
"Content-Transfer-Encoding: base64\r\n",
"Content-Disposition: attachment;\r\n",
/* Filename here */
"\r\n",
/* Encoded base64 contents here */
"\r\n",
"--------------030203080101020302070708--\r\n", // this type of boundary indicates that there are no more parts i = 15
"\r\n.\r\n",
NULL
};
Email::Email()
{
// empty constructor
}
void Email::setTo(const string to)
{
this->to = to;
}
void Email::setFrom(const string from)
{
this->from = from;
}
void Email::setCc(const string cc)
{
this->cc = cc;
}
void Email::setSubject(const string subject)
{
this->subject = subject;
}
void Email::setBody(const string body)
{
this->body = body;
}
void Email::setSMTP_username(const string user)
{
this->smtp_user = user;
}
void Email::setSMTP_password(const string pass)
{
this->smtp_password = pass;
}
void Email::setSMTP_host(const string hostname)
{
this->smtp_host = hostname;
}
void Email::addAttachment(const string file_path)
{
/*FILE* fp = NULL;
char buffer[MAX_LEN + 1] = { 0 };
char encodedBuffer[ENCODED_LEN] = { 0 };
char tempBuffer[MAX_LEN] = { 0 };
char* filename = NULL;
unsigned int fileSize = 0;
unsigned int bytesCopied = 0;
int bytesToCopy = 0;
int bytesRead = 0;
fp = fopen(file_path.c_str(), "rb");
if (fp) {
// get the file size
fseek(fp, 0, SEEK_END);
fileSize = ftell(fp);
fseek(fp, 0, SEEK_SET);
// copy the attachment header
string type(email_template[ATTACH_TYPE]);
this->attachments.push_back(type);
string encodingType(email_template[ATTACH_TRANSFER]);
this->attachments.push_back(encodingType);
string deposition(email_template[ATTACH_DEPOSITION]);
this->attachments.push_back(deposition);
// extract the filename from the path
filename = (char*)strrchr(file_path.c_str(), DIR_CHAR);
if (filename != NULL) {
filename += 1;
// push the filename
snprintf(tempBuffer, MAX_LEN, " filename=\"%s\"\r\n", filename);
string filename(tempBuffer);
this->attachments.push_back(filename);
string endLine(email_template[END_LINE]);
this->attachments.push_back(endLine);
// copy the file MAX_LEN bytes at a time into the attachments vector
while (bytesCopied < fileSize) {
// determine how many bytes to read
if (bytesCopied + MAX_LEN > fileSize) {
bytesToCopy = fileSize - bytesCopied;
}
else {
bytesToCopy = MAX_LEN;
}
// read from the file
memset(buffer, 0, MAX_LEN + 1);
bytesRead = fread(buffer, sizeof(char), bytesToCopy, fp);
// encoded the data read
memset(encodedBuffer, 0, ENCODED_LEN);
base64_encode(buffer, encodedBuffer, bytesRead);
// setup the encoded string so that we can push it to attachments
string line(encodedBuffer);
line += endLine;
this->attachments.push_back(line);
// update the number of bytes we have copied
bytesCopied += bytesToCopy;
}
this->attachments.push_back(endLine);
string boundary(email_template[BOUNDARY]);
this->attachments.push_back(boundary);
}
else {
removeAllAttachments();
cout << "Failed to extract filename!" << endl;
}
// close the file
fclose(fp);
}
else {
cout << "Unable to open file." << endl;
}
*/
}
void Email::constructEmail(void)
{
int i = 0;
char buffer[MAX_LEN];
string boundary(email_template[BOUNDARY]);
// time stuff
time_t rawtime;
struct tm* timeinfo;
// store all the email contents in a vector
// TO:
snprintf(buffer, MAX_LEN, "To: %s\r\n", this->to.c_str());
string line1(buffer);
this->email_contents.push_back(line1);
// FROM:
memset(buffer, 0, MAX_LEN);
snprintf(buffer, MAX_LEN, "From: %s\r\n", this->from.c_str());
string line2(buffer);
this->email_contents.push_back(line2);
// CC:
memset(buffer, 0, MAX_LEN);
snprintf(buffer, MAX_LEN, "Cc: %s\r\n", this->cc.c_str());
string line3(buffer);
if (this->cc.length() > 0) {
this->email_contents.push_back(line3);
}
// Subject:
memset(buffer, 0, MAX_LEN);
snprintf(buffer, MAX_LEN, "Subject: %s\r\n", this->subject.c_str());
string line4(buffer);
this->email_contents.push_back(line4);
// time:
time(&rawtime);
timeinfo = localtime(&rawtime);
memset(buffer, 0, MAX_LEN);
strftime(buffer, sizeof(buffer), "%d/%m/%Y %I:%M:%S +1100\r\n", timeinfo);
string time_str(buffer);
this->email_contents.push_back(time_str);
cout << time_str << endl;
for (i = 0; i < END_LINE; i++) { // other stuff e.g user-agent etc
string line(email_template[i]);
this->email_contents.push_back(line);
}
// add in the body
string endLine(email_template[END_LINE]);
this->email_contents.push_back(endLine);
memset(buffer, 0, MAX_LEN);
snprintf(buffer, MAX_LEN, "%s", this->body.c_str()); // Body:
string line5(buffer);
this->email_contents.push_back(line5); // body
this->email_contents.push_back(endLine); // \r\n
this->email_contents.push_back(boundary); // boundary
// add in the attachments
for (i = 0; i < attachments.size(); i++) {
this->email_contents.push_back(this->attachments[i]);
}
// add the last boundary with the two hyphens
string lastBoundary(email_template[END_OF_TRANSMISSION_BOUNDARY]);
this->email_contents.push_back(lastBoundary);
// specify that we don't want to send any more data
string endTransmission(email_template[END_OF_TRANSMISSION]);
this->email_contents.push_back(endTransmission);
}
/*
This function was taken and modified from:
https://curl.haxx.se/libcurl/c/smtp-ssl.html
*/
int Email::send(void) const
{
CURL* curl;
CURLcode res = CURLE_OK;
struct curl_slist* recipients = NULL;
struct upload_status upload_ctx;
upload_ctx.lines_read = 0;
curl = curl_easy_init();
global_vec = this->email_contents;
if (curl) {
/* Set username and password */
curl_easy_setopt(curl, CURLOPT_USERNAME, this->smtp_user.c_str());
curl_easy_setopt(curl, CURLOPT_PASSWORD, this->smtp_password.c_str());
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); // allows emails to be sent
/* This is the URL for your mailserver. Note the use of smtps:// rather
* than smtp:// to request a SSL based connection. */
curl_easy_setopt(curl, CURLOPT_URL, this->smtp_host.c_str());
/* If you want to connect to a site who isn't using a certificate that is
* signed by one of the certs in the CA bundle you have, you can skip the
* verification of the server's certificate. This makes the connection
* A LOT LESS SECURE.
*
* If you have a CA cert for the server stored someplace else than in the
* default bundle, then the CURLOPT_CAPATH option might come handy for
* you. */
#ifdef SKIP_PEER_VERIFICATION
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
#endif
/* If the site you're connecting to uses a different host name that what
* they have mentioned in their server certificate's commonName (or
* subjectAltName) fields, libcurl will refuse to connect. You can skip
* this check, but this will make the connection less secure. */
#ifdef SKIP_HOSTNAME_VERIFICATION
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
#endif
/* Note that this option isn't strictly required, omitting it will result
* in libcurl sending the MAIL FROM command with empty sender data. All
* autoresponses should have an empty reverse-path, and should be directed
* to the address in the reverse-path which triggered them. Otherwise,
* they could cause an endless loop. See RFC 5321 Section 4.5.5 for more
* details.
*/
curl_easy_setopt(curl, CURLOPT_MAIL_FROM, this->from.c_str());
/* Add two recipients, in this particular case they correspond to the
* To: and Cc: addressees in the header, but they could be any kind of
* recipient. */
recipients = curl_slist_append(recipients, this->to.c_str());
if (this->cc.length() > 0) {
recipients = curl_slist_append(recipients, this->cc.c_str());
}
curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, recipients);
/* We're using a callback function to specify the payload (the headers and
* body of the message). You could just use the CURLOPT_READDATA option to
* specify a FILE pointer to read from. */
curl_easy_setopt(curl, CURLOPT_READFUNCTION, payload_source);
curl_easy_setopt(curl, CURLOPT_READDATA, &upload_ctx);
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
/* Since the traffic will be encrypted, it is very useful to turn on debug
* information within libcurl to see what is happening during the
* transfer */
// curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
/* Send the message */
res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
/* Free the list of recipients */
curl_slist_free_all(recipients);
/* Always cleanup */
curl_easy_cleanup(curl);
}
return (int)res;
}
void Email::removeAllAttachments()
{
this->attachments.clear();
}
void Email::clearEmailContents()
{
this->email_contents.clear();
this->attachments.clear();
}
void Email::dump(void) const
{
int i = 0;
cout << "Email contents: " << endl;
for (i = 0; i < this->email_contents.size(); i++) {
cout << this->email_contents[i];
if (i == 20) {
break;
}
}
cout << "\n\nEmail attachments: " << endl;
for (i = 0; i < this->attachments.size(); i++) {
cout << this->attachments[i];
if (i == 5) {
break;
}
}
}
void base64_encode(char* input_buf, char* output_buf, size_t input_size)
{
char in[3];
char out[4];
size_t len = input_size;
*output_buf = 0;
for (size_t i = 0; i < len; )
{
int buf3_len = 0;
for (int j = 0; j < 3; ++j)
{
in[j] = input_buf[i++];
if (i > len)
in[j] = 0;
else
buf3_len++;
}
if (len > 0)
{
encodeblock(in, out, buf3_len);
strncat(output_buf, out, 4);
}
}
}
void encodeblock(char* in, char* out, int len)
{
out[0] = (char)b64_table[(int)(in[0] >> 2)];
out[1] = (char)b64_table[(int)(((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4))];
out[2] = (char)(len > 1 ? b64_table[(int)(((in[1] & 0x0f) << 2) | ((in[2] & 0xc0) >> 6))] : '=');
out[3] = (char)(len > 2 ? b64_table[(int)(in[2] & 0x3f)] : '=');
}
static size_t payload_source(void* ptr, size_t size, size_t nmemb, void* userp)
{
struct upload_status* upload_ctx = (struct upload_status*)userp;
const char* data;
if ((size == 0) || (nmemb == 0) || ((size * nmemb) < 1)) {
return 0;
}
if (upload_ctx->lines_read >= 0 && upload_ctx->lines_read < global_vec.size())
{
data = global_vec[upload_ctx->lines_read].c_str();
if (data) {
size_t len = strlen(data);
memcpy(ptr, data, len);
upload_ctx->lines_read++;
return len;
}
}
return 0;
}
main.cpp
#include <iostream>
#include "email.h"
using namespace std;
int main(void)
{
Email e;
int curlError = 0;
// e.dump();
e.setTo("receiver#example.com");
e.setFrom("sender#example.com");
e.setSubject("Test Email");
e.setCc("");
e.setBody("Do not reply to this email");
e.setSMTP_host("smtp://smtp.mail.com:587");
e.setSMTP_username("sender#example.com");
e.setSMTP_password("sender_password");
e.addAttachment("");
// e.addAttachment("email.h");
// e.addAttachment("main.cpp");
e.constructEmail();
e.dump();
curlError = e.send();
if (curlError) {
cout << "Error sending email!" << endl;
}
else {
cout << "Email sent successfully!" << endl;
}
return 0;
}
What is the reason this same code runs perfectly on one project but doesn't work on another?
EDIT:
error happens in Email::send(void) on the line :
global_vec2 = this->email_contents;
but I cannot try/catch it. it doesn't throws exception.
EDIT2 :
Problem is caused by static vector. It doesn't exist, doesn't initialize. I don't understand what's causing this project to prevent static std::vectors from being created.

json conversion error c++ IsString failed

I have data in a JSON object that I can't seem to get to.
Error:
> testGetprice3.o: testGetprice3.cpp:71: int getData(): Assertion
> `document["success"].IsString()' failed.
I have tried switching the datatypes, it only errors.
How do I correctly access my JSON object data?
#include "rapidjson/include/rapidjson/document.h"
#include <iostream>
#include <fstream>
#include <stdio.h>
#include <curl/curl.h>
#include <unistd.h>
#include <unordered_map>
#include <string>
using namespace rapidjson;
struct myData
{
std::fstream *file;
std::string *str;
};
size_t write_data(void *ptr, size_t size, size_t nmemb, myData *data)
{
size_t numBytes = size * nmemb;
if (data->file)
data->file->write((char*)ptr, numBytes);
if (data->str)
*data->str += std::string((char*)ptr, numBytes);
return numBytes;
}
//function to get coin data and perform analysis
int getData()
{
int count = 0;
//begin non terminating loop
while(true)
{
count++;
CURL *curl = curl_easy_init();
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, "http://pubapi.cryptsy.com/api.php?method=singlemarketdata&marketid=155");
std::fstream file("/home/coinz/cryptsy/myfile.txt", std::ios_base::out | std::ios_base::ate);
std::string json;
myData data;
data.file = &file;
data.str = &json;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
/* Perform the request, res will get the return code */
CURLcode res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK)
{
std::cerr << "curl_easy_perform() failed: " << curl_easy_strerror(res) << std::endl;
}
else
{
file << std::endl;
//begin deserialization
Document document;
document.Parse(json.c_str());
assert(document.HasMember("success"));
assert(document["success"].IsString());
//std::cout << "The Last Traded Price is = " << document["lasttradeprice"].GetString() << std::endl;
}
/* always cleanup */
curl_easy_cleanup(curl);
}
//timer for URL request. *ADUJST ME AS DESIRED*
usleep(10000000);
}
return 0;
}
//Le Main
int main(void)
{
getData();
}
Looking at your JSON data, the success member is an int (as it is not surrounded by quote marks).
More info on JSON types can be found here: http://en.wikipedia.org/wiki/JSON#Data_types.2C_syntax_and_example
If the success member was a string, the JSON data would look like this: {"success":"1",...
Therefore when you call assert(document["success"].IsString()) it causes the error, as the success member is not a string, but an int.
Since the success member is an int, change the assertion to test whether it's an int - assert(document["success"].IsInt())
You stated you changed datatypes, but did not state which ones you tried. From the URL you are using, success is being returned as "success":1, which is an int.
Change:
assert(document["success"].IsString());
to:
assert(document["success"].IsInt());

Segfault with multithreaded curl request

I'm having some trouble with a C++ program here. Basically I've written a simple wrapper for http requests, with the ability to do multiple requests at once.
Works absolutely fine, but when I do httpS requests, it crashes randomly in multithreaded mode. I'm using curl and posix threads.
Backtrace looks like this:
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x80996)[0x7fea9046d996]
/lib/x86_64-linux-gnu/libc.so.6(+0x82b80)[0x7fea9046fb80]
/lib/x86_64-linux-gnu/libc.so.6(realloc+0xf2)[0x7fea90470ae2]
/lib/x86_64-linux-gnu/libcrypto.so.1.0.0(CRYPTO_realloc+0x49)[0x7fea8f9c6169]
/lib/x86_64-linux-gnu/libcrypto.so.1.0.0(lh_insert+0x101)[0x7fea8fa4bfb1]
/lib/x86_64-linux-gnu/libcrypto.so.1.0.0(+0xe844e)[0x7fea8fa4e44e]
/lib/x86_64-linux-gnu/libcrypto.so.1.0.0(ERR_get_state+0xde)[0x7fea8fa4eeee]
/lib/x86_64-linux-gnu/libcrypto.so.1.0.0(ERR_clear_error+0x15)[0x7fea8fa4f065]
/usr/lib/x86_64-linux-gnu/libcurl.so.4(+0x24e79)[0x7fea90f10e79]
/usr/lib/x86_64-linux-gnu/libcurl.so.4(+0x39ea0)[0x7fea90f25ea0]
/usr/lib/x86_64-linux-gnu/libcurl.so.4(+0xf8fd)[0x7fea90efb8fd]
/usr/lib/x86_64-linux-gnu/libcurl.so.4(+0x219f5)[0x7fea90f0d9f5]
/usr/lib/x86_64-linux-gnu/libcurl.so.4(+0x35538)[0x7fea90f21538]
/usr/lib/x86_64-linux-gnu/libcurl.so.4(curl_multi_perform+0x91)[0x7fea90f21d31]
/usr/lib/x86_64-linux-gnu/libcurl.so.4(curl_easy_perform+0x107)[0x7fea90f19457]
./exbot[0x40273a]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x7f6e)[0x7fea90cd6f6e]
/lib/x86_64-linux-gnu/libc.so.6(clone+0x6d)[0x7fea904e79cd]
Could this be a bug in libcrypto?
Can I somehow tell curl not to use libcrypto? Any alternatives?
It only crahes wenn using httpS requests and works fine with even 10000 simultaneous http queries.
Cheers,
Thomas
Just for completeness my code:
// simple wrapper for http requests
#ifndef _REQUEST_H_
#define _REQUEST_H_
#include <curl/curl.h>
#include <pthread.h>
#include <string>
#include <iostream>
//////////////////////////////////
// MACROS
//////////////////////////////////
#define ERR(_msg) std::cerr << __FUNCTION__ << ": " << _msg << std::endl
//////////////////////////////////
// REQUEST WRAPPER
//////////////////////////////////
typedef unsigned int uint;
class RequestWrapper
{
private: // non copyable
RequestWrapper();
RequestWrapper(const RequestWrapper &that);
RequestWrapper &operator=(const RequestWrapper &that);
public:
struct Response
{
Response() : msg(""), success(false) {}
std::string msg;
bool success;
};
static Response simpleGET(std::string url, uint timeout);
static size_t write(char *content, size_t size, size_t nmemb, void *userp);
};
//////////////////////////////////
// GET
//////////////////////////////////
inline size_t RequestWrapper::write(char *content, size_t size, size_t nmemb, void *userp)
{
std::string *buf = static_cast<std::string *>(userp);
size_t realsize = size * nmemb;
for (uint i = 0; i < realsize; ++i)
{
buf->push_back(content[i]);
}
return realsize;
}
inline RequestWrapper::Response RequestWrapper::simpleGET(std::string url, uint timeout)
{
Response resp;
CURL *curl;
CURLcode res;
std::string buf;
// send request
buf.clear();
curl = curl_easy_init();
if (!curl)
{
//ERR("libcurl init failed");
return resp;
}
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast<void *>(&buf));
curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
res = curl_easy_perform(curl);
if(res != CURLE_OK)
{
//ERR("libcurl request failed, CODE: " << res);
return resp;
}
curl_easy_cleanup(curl);
// done
resp.msg = buf;
resp.success = true;
return resp;
}
//////////////////////////////////
// MULTITHREADED REQUEST
//////////////////////////////////
class RequestList
{
private:
std::vector<std::string> _reqs;
static void *sender(void *payload);
static pthread_mutex_t _mutex;
public:
inline void add(std::string request)
{
_reqs.push_back(request);
}
inline void clear()
{
_reqs.clear();
}
std::vector<std::string> send(uint timeout) const;
struct Payload
{
std::string url;
std::vector<std::string> *out;
uint tout, index;
Payload(std::string url,
std::vector<std::string> *out,
uint tout, uint index) : url(url), out(out), tout(tout), index(index) { }
Payload() : url(""), out(NULL), tout(0), index(0) { }
};
};
//////////////////////////////////
// SEND MT REQUEST
//////////////////////////////////
pthread_mutex_t RequestList::_mutex;
void *RequestList::sender(void *payload)
{
Payload *pl = static_cast<Payload *>(payload);
RequestWrapper::Response resp = RequestWrapper::simpleGET(pl->url, pl->tout);
pthread_mutex_lock(&_mutex);
if (resp.success)
{
pl->out->at(pl->index) = resp.msg;
std::cerr << ".";
}
else
{
std::cerr << "x";
}
pthread_mutex_unlock(&_mutex);
return NULL;
}
inline std::vector<std::string> RequestList::send(uint timeout) const
{
std::vector<std::string> resp;
resp.resize(_reqs.size());
Payload *payloads = new Payload[_reqs.size()];
pthread_t *tids = new pthread_t[_reqs.size()];
// create mutex
pthread_mutex_init(&_mutex, NULL);
// prepare payload and create thread
for (uint i = 0; i < _reqs.size(); ++i)
{
payloads[i] = Payload(_reqs[i], &resp, timeout, i);
pthread_create(&tids[i], NULL, RequestList::sender, static_cast<void *>(&payloads[i]));
}
// wait for threads to finish
for (uint i = 0; i < _reqs.size(); ++i)
{
pthread_join(tids[i], NULL);
}
std::cerr << std::endl;
//destroy mutex
pthread_mutex_destroy(&_mutex);
delete[] payloads;
delete[] tids;
return resp;
}
#endif
Libcrypto is part of OpenSSL, which is not thread-safe unless you provide the necessary callbacks. According to the documentation, on a POSIX-compliant system (which has thread-local errno) the default thread-id implementation is acceptable, so you just need a locking function:
void locking_function(int mode, int n, const char *file, int line);
This function will need to maintain a set of CRYPTO_num_locks() mutexes, and lock or unlocks the n-th mutex depending on the value of mode. You can read the documentation for more details. The libcurl website actually has some sample code showing how to do this.
Alternatively, you can build libcurl with a different SSL library that is thread safe, such as GnuTLS.

Can one read a remote file as an istream with libcurl?

I'd like to use the libcurl library to open a remote date file and iterate through it with an istream. I've looked through the nice example in this thread but it writes the remote file to a local file. Instead I'd like to have the remote reads be pushed to an istream for subsequent programmatic manipulation. Is this possible? I would greatly appreciate help.
Best,
Aaron
Boost's IO Stream might be a better solution than STL's own stream. At least it is much simpler to create a boost stream. From boost's own docs:
#include <curl/curl.h>
#include <boost/iostreams/stream.hpp>
class CURLDevice
{
private:
CURL* handle;
public:
typedef char char_type;
typedef boost::iostreams::source_tag category;
CURLDevice()
{
handle = curl_easy_init();
}
CURLDevice(const std::string &url)
{
handle = curl_easy_init();
open( url );
}
~CURLDevice()
{
curl_easy_cleanup(handle);
}
void open(const std::string &url)
{
curl_easy_setopt(handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(handle, CURLOPT_CONNECT_ONLY, 1);
curl_easy_perform(handle);
}
std::streamsize read(char* s, std::streamsize n)
{
size_t read;
CURLcode ret = curl_easy_recv(handle, s, n, &read);
if ( ret == CURLE_OK || ret == CURLE_AGAIN )
return read;
else
return -1;
}
};
typedef boost::iostreams::stream<CURLDevice> CURLStream;
int main(int argc, char **argv)
{
curl_global_init(CURL_GLOBAL_ALL);
{
CURLStream stream("http://google.com");
char buffer[256];
int sz;
do
{
sz = 256;
stream.read( buffer, sz );
sz = stream.gcount();
std::cout.write( buffer, sz );
}
while( sz > 0 );
}
curl_global_cleanup();
return 0;
}
Note: when I run the code above I get a segfault in CURL, this appears to be because I don't know exactly how to use curl itself.