I'm trying to make a CurlResponse object encapsulating libcurl response. My implementation of curl options WRITEFUNCTION and HEADERFUNCTION is mostly the same, the only difference being that in first case I'm calling response->appendBody and in the second - response->appendHeader. I would like to have one function and pass a pointer to appropriate method as a parameter, e.g. WRITEDATA would be response->appendBody, and I could call writer(data). However when I execute the below code, I get an error:
error: cannot pass objects of non-trivially-copyable type ‘struct std::_Bind<std::_Mem_fn<void (CurlResponse::*)(std::basic_string<char>)>(CurlResponse*, std::_Placeholder<1>)>’ through ‘...’
...
#include <iostream>
#include <string>
#include <functional>
#include <curl/curl.h>
using namespace std;
class CurlResponse {
public:
void appendBody(string data) {
cout << "Append body " << data << endl;
}
void appendHeader(string data) {
cout << "Append header " << data << endl;
}
};
//size_t WriteMemoryCallback(char * contents, size_t size, size_t nmemb, CurlResponse* response)
size_t WriteMemoryCallback(char * contents, size_t size, size_t nmemb, function<void(string)> writer)
{
size_t realsize = size * nmemb;
string data(contents, realsize);
// response->appendBody(data);
writer(data);
return realsize;
}
size_t WriteHeaderCallback(char * contents, size_t size, size_t nmemb, CurlResponse* response)
{
size_t realsize = size * nmemb;
string data(contents, realsize);
response->appendHeader(data);
return realsize;
}
int main() {
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if (! curl) return 1;
curl_easy_setopt(curl, CURLOPT_URL, "http://localhost");
CurlResponse* response = new CurlResponse();
auto writeBody = std::bind(&CurlResponse::appendBody, response, placeholders::_1);
writeBody("Test writing to body");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, writeBody);
// curl_easy_setopt(curl, CURLOPT_WRITEDATA, response);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, WriteHeaderCallback);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, response);
res = curl_easy_perform(curl);
if(res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
}
curl_easy_cleanup(curl);
return 0;
}
How do I fix this? Does this mean I can use std::binded functions in the same function, but can't pass them anywhere?
The problem is that you are trying to pass a complex object through ..., as the compiler already tells you. The probably best solution is to first wrap the std::bind in a std::function object, to avoid having to repeat the complete type:
function<void(string)> writeBodyPass(writeBody);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &writeBodyPass);
You then have to fix the signature (and body) of your callback, as you are now receiving a pointer to the std::function object:
size_t WriteMemoryCallback(char * contents, size_t size, size_t nmemb, function<void(string)> *writer)
{
size_t realsize = size * nmemb;
string data(contents, realsize);
(*writer)(data);
return realsize;
}
Related
I am trying to collect some data from a URL. If I do not define any CURLOPT_WRITEFUNCTION and CURLOPT_WRITEDATA I can obviously see the output on console. Then I tried to write that data to memory by copiying the example code, however userdata argument of my callback function returned NULL and I got following exception on line:
char* ptr = (char*)realloc(mem->memory, mem->size + realsize + 1);
Exception thrown: read access violation.
mem was nullptr.
Am I doing something wrong?
Here is my code:
struct MemoryStruct {
char* memory;
size_t size;
};
//-----------------
// Curl's callback
//-----------------
size_t CurlWrapper::curl_cb(char* data, size_t size, size_t nmemb, void* response)
{
size_t realsize = size * nmemb;
std::cout << "CALLBACK CALLED" << std::endl;
MemoryStruct* mem = (struct MemoryStruct*)response;
char* ptr = (char*)realloc(mem->memory, mem->size + realsize + 1);
if (!ptr) {
/* out of memory! */
printf("not enough memory (realloc returned NULL)\n");
return 0;
}
mem->memory = ptr;
memcpy(&(mem->memory[mem->size]), data, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
return realsize;
}
//--------------------
// Do the curl
//--------------------
void CurlWrapper::curl_api(
const std::string& url,
std::string& str_result)
{
MemoryStruct chunk;
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &CurlWrapper::curl_cb);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&chunk);
// TODO: enable ssh certificate
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false); // true
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, false); // 2
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "zlib");
auto res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK) {
// nothing
std::cout << "curl_easy_perform() failed: " << curl_easy_strerror(res) << std::endl;
}
}
}
libcurl version: 7.82.0
Since libcurl is a C library, it does not know anything about C++ member functions or objects. You can overcome this "limitation" with relative ease using for example a static member function that is passed a pointer to the class.
See this example (from the everything curl book).
// f is the pointer to your object.
static size_t YourClass::func(void *buffer, size_t sz, size_t n, void *f)
{
// Call non-static member function.
static_cast<YourClass*>(f)->nonStaticFunction();
}
// This is how you pass pointer to the static function:
curl_easy_setopt(hcurl, CURLOPT_WRITEFUNCTION, YourClass::func);
curl_easy_setopt(hcurl, CURLOPT_WRITEDATA, this);
we need to receive response string from curl in CPP ,I tried following options but nothing worked.js uses xhr.responseText for this.I need to do in cpp.
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
size_t realsize = size * nmemb;
const char* sp = static_cast<const char*>(contents);
readBuffer.append(sp, realsize);
return realsize;
}
CURLcode res;
char* http_error= new char[100];
readBuffer.clear();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
CURLcode code(CURLE_FAILED_INIT);
code = curl_easy_perform(curl);
cout << "Curl response msg CURLOPT_WRITEDATA: "<<curl_easy_strerror(code)<< " respose :"<<readBuffer;
res=curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code) ;
cout << "Curl response msg: "<< curl_easy_strerror(res);
Change your WriteCallback function to this:
size_t WriteCallback(void* contents, size_t size, size_t nmemb, std::string* userp) {
userp->append((char*) contents, size * nmemb);
return size * nmemb;
}
Remember, you are passing in &readBuffer as the CURL_WRITEDATA option. This shows up as the fourth parameter in the WriteCallback. Since the type of &readBuffer is std::string*, you can use that as the signature in your callback.
Additionally, since it's a std::string* and not a std::string, you have to access append through the pointer, hence the -> instead of .. After curl_easy_perform, readBuffer should hold the response from the curl request.
To get the response code, you can grab that after making the request:
long http_code;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
I'm trying to download image file from a url. I followed the example using fwrite and it succeeded. Now I'm trying to use fstream::write to save the data (ios::binary), but the data is damaged. Here is my code:
#include"stdafx.h"
#include<fstream>
#include<iostream>
#include <curl/curl.h>
#include <string.h>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, char* out) {
//void *ptr, size_t size, size_t nmemb, File* fp
fstream file;
if (file.is_open()){
file.close();
file.clear();
}
file.open(out, ios::out | ios::binary);
if (file.is_open()){
cout << "open successfully\n" << endl;
file.write((char*)ptr, nmemb*size); // Does it correct?
};
// fwrite(ptr,size,nmemb,fp);
file.close();
file.clear();
cout <<"\n sizeof(ptr): " << sizeof(ptr) //size of ptr[0]?
<<"\n sizeof(char): " << sizeof(char)
<<"\n size: " << size
<<"\n nmemb: " << nmemb<< endl;
return size*nmemb;
}
I'm confused about the parameters in write_data.
According to CURLOPT_WRITEFUNCTION
size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata);
"ptr points to the delivered data, and the size of that data is size multiplied with nmemb."
......so what are the meanings of size and nmemb?
When tried to download the data from a website,I printed first 3 parameters. It seems that char*ptr is the memory address that data stored (as 'char a[]'?), and size is element's size, nmemb is number of the elements. So the data size = size * nmemb. Am I correct?
The output is confusing too:
open successfully
sizeof(ptr):4
sizeof(char):1
size:1
nmemb:2715
open successfully
sizeof(ptr):4
sizeof(char):1
size:1
nmemb:4865
download successfully
When download the same url, nmemb and files' open times often change.
I'm also confused about 'sizeof(ptr)', it returns '4'(size of int?). How can I use the 'sizeof' to get the size of data memory, so that I can proof that data size is 'size * nmemb'?
CURLcode download(char* url,char* out){
CURL *curl = NULL;
//FILE *fp = NULL;
CURLcode res;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, out); //fp
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
return res;
}
else
{
return CURLE_FAILED_INIT;
}
}
int main()
{
CURLcode res = download("http://XXXXXX.gif", "D:\\test.gif");
if (CURLE_OK == res)
cout << "download successfully.\n" << endl;
else
cout<<"cannot download.\n"<<endl;
return 0;
}
Thanks! :)
This callback can be multiple times per file. You should NOT create a new file stream every time the function is called - you should pass it in using the user data parameter. Otherwise you will just keep overwriting the data at the beginning of the file.
Here is an example implementation:
size_t write_data(char *ptr, size_t size, size_t nmemb, void *userdata)
{
std::ofstream *out = static_cast<std::ofstream *>(userdata);
size_t nbytes = size * nmemb;
out->write(ptr, nbytes);
return nbytes;
}
You also need to adjust the call to curl_easy_setopt with the parameter CURLOPT_WRITEDATA to actually pass your file stream. Make sure that the stream does not go out of scope while the functions run!
CURLcode download(char* url, char* out) {
CURL *curl = NULL;
std::ofstream output(out, ios::binary);
CURLcode res;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &output);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
return res;
}
else
{
return CURLE_FAILED_INIT;
}
}
I really can not understand what my problem
Here is my code:
#include <iostream>
#include <curl/curl.h>
#include <boost/regex.hpp>
using namespace std;
using namespace boost;
int Write_callback(char *data, size_t size, size_t nmemb, string *buffer)
{
int result = 0;
if (buffer != NULL)
{
buffer->append(data, size * nmemb);
result = size * nmemb;
}
return result;
}
char *Get(string url)
{
CURL *curl;
CURLcode res;
char *readBuffer;
int retries = 2;
curl = curl_easy_init();
if (curl) {
do {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10);
curl_easy_setopt(curl, CURLOPT_COOKIEFILE, "cookies.txt");
curl_easy_setopt(curl, CURLOPT_COOKIEJAR, "cookies.txt");
curl_easy_setopt(curl, CURLOPT_USERAGENT,
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.5");
res = curl_easy_perform(curl);
} while (CURLE_OK != res && retries--);
curl_easy_cleanup(curl);
return readBuffer;
}
}
int main(int argc, char** argv)
{
char *www = Get("google.com");
string Str = "dsfgdasgadgasdg";
boost::regex RegEx("(.+)");
boost::smatch Results;
boost::regex_match(Str, Results, RegEx);
cout << "Print entire match:\n " << Results[1] << endl;
return 0;
}
This compiles, but when you run an error:
terminate called after throwing an instance of 'std::length_error'
what(): basic_string::_S_create
If I comment out this:
char *www = Get("google.com");
That works
If I comment out this:
string Str = "dsfgdasgadgasdg";
boost::regex RegEx("(.+)");
boost::smatch Results;
boost::regex_match(Str, Results, RegEx);
cout << "Print entire match:\n " << Results[1] << endl;
That works too..
Why it does not work together ??
In your Get function you declare the local variable readBuffer character pointer:
char *readBuffer;
You use the address of that as the userdata for your Write_callback function. So the userdata is a char**. But in your Write_Callback you have the userdata field a a std::string:
int Write_callback(char *data, size_t size, size_t nmemb, string *buffer)
With LibCurl the callback has the userdata parameter as a void* so it isn't a std::string:
size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata);
See http://curl.haxx.se/libcurl/c/CURLOPT_WRITEFUNCTION.html
Pass a std::string pointer as the userdata instead of a char**. And I would keep the Write_callback parameter a void* and cast that void* to a std::string* in the callback.
Hope that helps!
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
_getch();
return 0;
}
string contents = "";
I would like to save the result of the curl html content in a string, how do I do this?
It's a silly question but unfortunately, I couldn't find anywhere in the cURL examples for C++
thanks!
You will have to use CURLOPT_WRITEFUNCTION to set a callback for writing. I can't test to compile this right now, but the function should look something close to;
static std::string readBuffer;
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
size_t realsize = size * nmemb;
readBuffer.append(contents, realsize);
return realsize;
}
Then call it by doing;
readBuffer.clear();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
// ...other curl options
res = curl_easy_perform(curl);
After the call, readBuffershould have your contents.
Edit: You can use CURLOPT_WRITEDATA to pass the buffer string instead of making it static. In this case I just made it static for simplicity. A good page to look (besides the linked example above) is here for an explanation of the options.
Edit2: As requested, here's a complete working example without the static string buffer;
#include <iostream>
#include <string>
#include <curl/curl.h>
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
std::string readBuffer;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
std::cout << readBuffer << std::endl;
}
return 0;
}
On my blog I have published a simple wrapper class to perform this task.
Usage example:
#include "HTTPDownloader.hpp"
int main(int argc, char** argv) {
HTTPDownloader downloader;
std::string content = downloader.download("https://stackoverflow.com");
std::cout << content << std::endl;
}
Here's the header file:
/**
* HTTPDownloader.hpp
*
* A simple C++ wrapper for the libcurl easy API.
*
* Written by Uli Köhler (techoverflow.net)
* Published under CC0 1.0 Universal (public domain)
*/
#ifndef HTTPDOWNLOADER_HPP
#define HTTPDOWNLOADER_HPP
#include <string>
/**
* A non-threadsafe simple libcURL-easy based HTTP downloader
*/
class HTTPDownloader {
public:
HTTPDownloader();
~HTTPDownloader();
/**
* Download a file using HTTP GET and store in in a std::string
* #param url The URL to download
* #return The download result
*/
std::string download(const std::string& url);
private:
void* curl;
};
#endif /* HTTPDOWNLOADER_HPP */
Here's the source code:
/**
* HTTPDownloader.cpp
*
* A simple C++ wrapper for the libcurl easy API.
*
* Written by Uli Köhler (techoverflow.net)
* Published under CC0 1.0 Universal (public domain)
*/
#include "HTTPDownloader.hpp"
#include <curl/curl.h>
#include <curl/easy.h>
#include <curl/curlbuild.h>
#include <sstream>
#include <iostream>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
string data((const char*) ptr, (size_t) size * nmemb);
*((stringstream*) stream) << data;
return size * nmemb;
}
HTTPDownloader::HTTPDownloader() {
curl = curl_easy_init();
}
HTTPDownloader::~HTTPDownloader() {
curl_easy_cleanup(curl);
}
string HTTPDownloader::download(const std::string& url) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
/* example.com is redirected, so we tell libcurl to follow redirection */
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); //Prevent "longjmp causes uninitialized stack frame" bug
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "deflate");
std::stringstream out;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out);
/* Perform the request, res will get the return code */
CURLcode res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
return out.str();
}
Using the 'new' C++11 lambda functionality, this can be done in a few lines of code.
#ifndef WIN32 #define __stdcall "" #endif //For compatibility with both Linux and Windows
std::string resultBody { };
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &resultBody);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, static_cast<size_t (__stdcall *)(char*, size_t, size_t, void*)>(
[](char* ptr, size_t size, size_t nmemb, void* resultBody){
*(static_cast<std::string*>(resultBody)) += std::string {ptr, size * nmemb};
return size * nmemb;
}
));
CURLcode curlResult = curl_easy_perform(curl);
std::cout << "RESULT BODY:\n" << resultBody << std::endl;
// Cleanup etc
Note the __stdcall cast is needed to comply to the C calling convention (cURL is a C library)
This might not work right away but should give you an idea:
#include <string>
#include <curl.h>
#include <stdio.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main() {
std::string tempname = "temp";
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
FILE *fp = fopen(tempname.c_str(),"wb");
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
fp = fopen(tempname.c_str(),"rb");
fseek (fp , 0 , SEEK_END);
long lSize = ftell (fp);
rewind(fp);
char *buffer = new char[lSize+1];
fread (buffer, 1, lSize, fp);
buffer[lSize] = 0;
fclose(fp);
std::string content(buffer);
delete [] buffer;
}
}
Came out with useful, yet simple solution, which overloads std::ostream::operator<<
#include <ostream>
#include <curl/curl.h>
size_t curlCbToStream (
char * buffer,
size_t nitems,
size_t size,
std::ostream * sout
)
{
*sout << buffer;
return nitems * size;
}
std::ostream & operator<< (
std::ostream & sout,
CURL * request
)
{
::curl_easy_setopt(request, CURLOPT_WRITEDATA, & sout);
::curl_easy_setopt(request, CURLOPT_WRITEFUNCTION, curlCbToStream);
::curl_easy_perform(request);
return sout;
}
Possible drawback of taken approach could be:
typedef void CURL;
That means it covers all known pointer types.
Based on #JoachimIsaksson answer, here is a more verbose output that handles out-of-memory and has a limit for the maximum output from curl (as CURLOPT_MAXFILESIZE limits only based on header information and not on the actual size transferred ).
#DEFINE MAX_FILE_SIZE = 10485760 //10 MiB
size_t curl_to_string(void *ptr, size_t size, size_t count, void *stream)
{
if(((string*)stream)->size() + (size * count) > MAX_FILE_SIZE)
{
cerr<<endl<<"Could not allocate curl to string, output size (current_size:"<<((string*)stream)->size()<<"bytes + buffer:"<<(size * count) << "bytes) would exceed the MAX_FILE_SIZE ("<<MAX_FILE_SIZE<<"bytes)";
return 0;
}
int retry=0;
while(true)
{
try{
((string*)stream)->append((char*)ptr, 0, size*count);
break;// successful
}catch (const std::bad_alloc&) {
retry++;
if(retry>100)
{
cerr<<endl<<"Could not allocate curl to string, probably not enough memory, aborting after : "<<retry<<" tries at 10s apart";
return 0;
}
cerr<<endl<<"Could not allocate curl to string, probably not enough memory, sleeping 10s, try:"<<retry;
sleep(10);
}
}
return size*count;
}
I use Joachim Isaksson's answer with a modern C++ adaptation of CURLOPT_WRITEFUNCTION.
No nagging by the compiler for C-style casts.
static auto WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) -> size_t {
static_cast<string*>(userdata)->append(ptr, size * nmemb);
return size * nmemb;
}