Save cURL content result into a string in C++ - c++

Question

int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
_getch();
return 0;
}
string contents = "";
I would like to save the result of the curl html content in a string, how do I do this?
It's a silly question but unfortunately, I couldn't find anywhere in the cURL examples for C++
thanks!

Answer 1

You will have to use CURLOPT_WRITEFUNCTION to set a callback for writing. I can't test to compile this right now, but the function should look something close to;
static std::string readBuffer;
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
size_t realsize = size * nmemb;
readBuffer.append(contents, realsize);
return realsize;
}
Then call it by doing;
readBuffer.clear();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
// ...other curl options
res = curl_easy_perform(curl);
After the call, readBuffershould have your contents.
Edit: You can use CURLOPT_WRITEDATA to pass the buffer string instead of making it static. In this case I just made it static for simplicity. A good page to look (besides the linked example above) is here for an explanation of the options.
Edit2: As requested, here's a complete working example without the static string buffer;
#include <iostream>
#include <string>
#include <curl/curl.h>
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
std::string readBuffer;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
std::cout << readBuffer << std::endl;
}
return 0;
}

Answer 2

On my blog I have published a simple wrapper class to perform this task.
Usage example:
#include "HTTPDownloader.hpp"
int main(int argc, char** argv) {
HTTPDownloader downloader;
std::string content = downloader.download("https://stackoverflow.com");
std::cout << content << std::endl;
}
Here's the header file:
/**
* HTTPDownloader.hpp
*
* A simple C++ wrapper for the libcurl easy API.
*
* Written by Uli Köhler (techoverflow.net)
* Published under CC0 1.0 Universal (public domain)
*/
#ifndef HTTPDOWNLOADER_HPP
#define HTTPDOWNLOADER_HPP
#include <string>
/**
* A non-threadsafe simple libcURL-easy based HTTP downloader
*/
class HTTPDownloader {
public:
HTTPDownloader();
~HTTPDownloader();
/**
* Download a file using HTTP GET and store in in a std::string
* #param url The URL to download
* #return The download result
*/
std::string download(const std::string& url);
private:
void* curl;
};
#endif /* HTTPDOWNLOADER_HPP */
Here's the source code:
/**
* HTTPDownloader.cpp
*
* A simple C++ wrapper for the libcurl easy API.
*
* Written by Uli Köhler (techoverflow.net)
* Published under CC0 1.0 Universal (public domain)
*/
#include "HTTPDownloader.hpp"
#include <curl/curl.h>
#include <curl/easy.h>
#include <curl/curlbuild.h>
#include <sstream>
#include <iostream>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
string data((const char*) ptr, (size_t) size * nmemb);
*((stringstream*) stream) << data;
return size * nmemb;
}
HTTPDownloader::HTTPDownloader() {
curl = curl_easy_init();
}
HTTPDownloader::~HTTPDownloader() {
curl_easy_cleanup(curl);
}
string HTTPDownloader::download(const std::string& url) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
/* example.com is redirected, so we tell libcurl to follow redirection */
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); //Prevent "longjmp causes uninitialized stack frame" bug
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "deflate");
std::stringstream out;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out);
/* Perform the request, res will get the return code */
CURLcode res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
return out.str();
}

Answer 3

Using the 'new' C++11 lambda functionality, this can be done in a few lines of code.
#ifndef WIN32 #define __stdcall "" #endif //For compatibility with both Linux and Windows
std::string resultBody { };
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &resultBody);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, static_cast<size_t (__stdcall *)(char*, size_t, size_t, void*)>(
[](char* ptr, size_t size, size_t nmemb, void* resultBody){
*(static_cast<std::string*>(resultBody)) += std::string {ptr, size * nmemb};
return size * nmemb;
}
));
CURLcode curlResult = curl_easy_perform(curl);
std::cout << "RESULT BODY:\n" << resultBody << std::endl;
// Cleanup etc
Note the __stdcall cast is needed to comply to the C calling convention (cURL is a C library)

Answer 4

This might not work right away but should give you an idea:
#include <string>
#include <curl.h>
#include <stdio.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main() {
std::string tempname = "temp";
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
FILE *fp = fopen(tempname.c_str(),"wb");
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
fp = fopen(tempname.c_str(),"rb");
fseek (fp , 0 , SEEK_END);
long lSize = ftell (fp);
rewind(fp);
char *buffer = new char[lSize+1];
fread (buffer, 1, lSize, fp);
buffer[lSize] = 0;
fclose(fp);
std::string content(buffer);
delete [] buffer;
}
}

Answer 5

Came out with useful, yet simple solution, which overloads std::ostream::operator<<
#include <ostream>
#include <curl/curl.h>
size_t curlCbToStream (
char * buffer,
size_t nitems,
size_t size,
std::ostream * sout
)
{
*sout << buffer;
return nitems * size;
}
std::ostream & operator<< (
std::ostream & sout,
CURL * request
)
{
::curl_easy_setopt(request, CURLOPT_WRITEDATA, & sout);
::curl_easy_setopt(request, CURLOPT_WRITEFUNCTION, curlCbToStream);
::curl_easy_perform(request);
return sout;
}
Possible drawback of taken approach could be:
typedef void CURL;
That means it covers all known pointer types.

Answer 6

Based on #JoachimIsaksson answer, here is a more verbose output that handles out-of-memory and has a limit for the maximum output from curl (as CURLOPT_MAXFILESIZE limits only based on header information and not on the actual size transferred ).
#DEFINE MAX_FILE_SIZE = 10485760 //10 MiB
size_t curl_to_string(void *ptr, size_t size, size_t count, void *stream)
{
if(((string*)stream)->size() + (size * count) > MAX_FILE_SIZE)
{
cerr<<endl<<"Could not allocate curl to string, output size (current_size:"<<((string*)stream)->size()<<"bytes + buffer:"<<(size * count) << "bytes) would exceed the MAX_FILE_SIZE ("<<MAX_FILE_SIZE<<"bytes)";
return 0;
}
int retry=0;
while(true)
{
try{
((string*)stream)->append((char*)ptr, 0, size*count);
break;// successful
}catch (const std::bad_alloc&) {
retry++;
if(retry>100)
{
cerr<<endl<<"Could not allocate curl to string, probably not enough memory, aborting after : "<<retry<<" tries at 10s apart";
return 0;
}
cerr<<endl<<"Could not allocate curl to string, probably not enough memory, sleeping 10s, try:"<<retry;
sleep(10);
}
}
return size*count;
}

Answer 7

I use Joachim Isaksson's answer with a modern C++ adaptation of CURLOPT_WRITEFUNCTION.
No nagging by the compiler for C-style casts.
static auto WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) -> size_t {
static_cast<string*>(userdata)->append(ptr, size * nmemb);
return size * nmemb;
}