I wish to write a C++ program to save the returned header to a variable and save the returned body to a text file. How can I do this?
Currently, my approach is to overload the handleData function, but the compiler returns the error overloaded function with no contextual type information. This is what I have written so far (an extract of the code):
static size_t handleData(char *ptr, size_t size, size_t nmemb, string *str){
string temp = string(ptr);
// catch the cookie
if (temp.substr(0,10)=="Set-Cookie"){
*str = temp;
}
return size * nmemb;
}
static size_t handleData(char *ptr, size_t size, size_t nmemb, FILE *stream){
int written = fwrite(ptr, size, nmemb, stream);
return written;
}
FILE *bodyfile;
string *return_header = new string;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, handleData);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, return_header);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, bodyfile);
You should use CURLOPT_HEADERFUNCTION instead.
static size_t handleHeader(char *ptr, size_t size, size_t nmemb, string *str){
// ...
}
static size_t handleData(char *ptr, size_t size, size_t nmemb, FILE *stream){
// ...
}
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, handleHeader);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, handleData);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, return_header);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, bodyfile);
Related
I want to extract header information by using the CURLOPT_HEADERFUNCTION in my c++ program.
How can I use CURLOPT_HEADERFUNCTION to read a single response header field? provides the solution on how to get those header information but I want to know why my code is not working and a possible solution with example.
//readHeader function which returns the specific header information
size_t readHeader(char* header, size_t size, size_t nitems, void *userdata) {
Erza oprations; //class which contains string function like startsWith etc
if (oprations.startsWith(header, "Content-Length:")) {
std::string header_in_string = oprations.replaceAll(header, "Content-Length:", "");
long size = atol(header_in_string.c_str());
file_size = size; // file_size is global variable
std::cout << size; // here it is showing correct file size
}
else if (oprations.startsWith(header, "Content-Type:")) {
// do something
}else
// do something
return size * nitems;
}
// part of main function
curl = curl_easy_init();
if (curl) {
fp = fopen(path, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_CAINFO, "./ca-bundle.crt");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, false);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, readHeader);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
std::cout << file_size; // showing value 0
Getting correct file size in readHeader function but getting 0 bytes in main function.
As shown in your github depot, oprations (operations !?) is a local variable, and will be released at the end of the readHeader function. A way to process the readHeader function and get the correct file size for a given Erza instance is to pass its pointer to userdata value. The Erza class may be rewritten as :
class Erza : public Endeavour {
//... your class body
public:
bool download (const char *url,const char* path){
curl = curl_easy_init();
if (curl) {
fp = fopen(path, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_CAINFO, "./ca-bundle.crt");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, false);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, readHeader);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, this ); //<-- set this pointer to userdata value used in the callback.
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
return false;
}else
return true;
}
size_t analyseHeader( char* header, size_t size, size_t nitems ){
if (startsWith(header, "Content-Length:")) {
std::string header_in_string = replaceAll(header, "Content-Length:", "");
long size = atol(header_in_string.c_str());
file_size = size; // file_size is a member variable
std::cout << size; // here it is showing correct file size
}
else if (startsWith(header, "Content-Type:")) {
// do something
}else
// do something
return size * nitems;
}
}//Eof class Erza
size_t readHeader(char* header, size_t size, size_t nitems, void *userdata) {
//get the called context (Erza instance pointer set in userdata)
Erza * oprations = (Erza *)userdata;
return oprations->analyseHeader( header, size, nitems );
}
I am trying to download a .txt file from a server which I can access via the web browser on my raspberry pi.
Curl library gives segmentation error when I am trying to do this. Here is the code I am using.
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
int checkNewFiles(){
CURL *curl;
FILE *fp;
CURLcode res;
string url = "http://52.233.176.151:1880/files/device/software/text.txt";
char outfilename[FILENAME_MAX] = "/home/pi/Desktop/project/cpp/ab.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
I found the problem, what is url.c_str() doing?
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
change this to
curl_easy_setopt(curl, CURLOPT_URL, url);
Example : Curl program that download the text file.
Offcourse you need to add this neccessary header file here.
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main(void) {
CURL *curl;
FILE *fp;
CURLcode res;
const char *url = "http://localhost/yourfile.txt";
char outfilename[FILENAME_MAX] = "C:\\outfile.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); /* enable failure on http errors */
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
if(res != CURLE_OK) { /* check that the operation was successful */
printf("curl_easy_perform(): %s\n", curl_easy_strerror(res));
}
/* always cleanup */
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
I noticed you're not checking for errors after fopen. If it fails, it returns a NULL pointer, which would cause a segfault when curl attempts to write to it.
I'm not convinced that c_str() was the culprit to your segfault in the original question as I have used that in numerous applications with no problems.
we need to receive response string from curl in CPP ,I tried following options but nothing worked.js uses xhr.responseText for this.I need to do in cpp.
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
size_t realsize = size * nmemb;
const char* sp = static_cast<const char*>(contents);
readBuffer.append(sp, realsize);
return realsize;
}
CURLcode res;
char* http_error= new char[100];
readBuffer.clear();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
CURLcode code(CURLE_FAILED_INIT);
code = curl_easy_perform(curl);
cout << "Curl response msg CURLOPT_WRITEDATA: "<<curl_easy_strerror(code)<< " respose :"<<readBuffer;
res=curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code) ;
cout << "Curl response msg: "<< curl_easy_strerror(res);
Change your WriteCallback function to this:
size_t WriteCallback(void* contents, size_t size, size_t nmemb, std::string* userp) {
userp->append((char*) contents, size * nmemb);
return size * nmemb;
}
Remember, you are passing in &readBuffer as the CURL_WRITEDATA option. This shows up as the fourth parameter in the WriteCallback. Since the type of &readBuffer is std::string*, you can use that as the signature in your callback.
Additionally, since it's a std::string* and not a std::string, you have to access append through the pointer, hence the -> instead of .. After curl_easy_perform, readBuffer should hold the response from the curl request.
To get the response code, you can grab that after making the request:
long http_code;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
I'm trying to make a CurlResponse object encapsulating libcurl response. My implementation of curl options WRITEFUNCTION and HEADERFUNCTION is mostly the same, the only difference being that in first case I'm calling response->appendBody and in the second - response->appendHeader. I would like to have one function and pass a pointer to appropriate method as a parameter, e.g. WRITEDATA would be response->appendBody, and I could call writer(data). However when I execute the below code, I get an error:
error: cannot pass objects of non-trivially-copyable type ‘struct std::_Bind<std::_Mem_fn<void (CurlResponse::*)(std::basic_string<char>)>(CurlResponse*, std::_Placeholder<1>)>’ through ‘...’
...
#include <iostream>
#include <string>
#include <functional>
#include <curl/curl.h>
using namespace std;
class CurlResponse {
public:
void appendBody(string data) {
cout << "Append body " << data << endl;
}
void appendHeader(string data) {
cout << "Append header " << data << endl;
}
};
//size_t WriteMemoryCallback(char * contents, size_t size, size_t nmemb, CurlResponse* response)
size_t WriteMemoryCallback(char * contents, size_t size, size_t nmemb, function<void(string)> writer)
{
size_t realsize = size * nmemb;
string data(contents, realsize);
// response->appendBody(data);
writer(data);
return realsize;
}
size_t WriteHeaderCallback(char * contents, size_t size, size_t nmemb, CurlResponse* response)
{
size_t realsize = size * nmemb;
string data(contents, realsize);
response->appendHeader(data);
return realsize;
}
int main() {
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if (! curl) return 1;
curl_easy_setopt(curl, CURLOPT_URL, "http://localhost");
CurlResponse* response = new CurlResponse();
auto writeBody = std::bind(&CurlResponse::appendBody, response, placeholders::_1);
writeBody("Test writing to body");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, writeBody);
// curl_easy_setopt(curl, CURLOPT_WRITEDATA, response);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, WriteHeaderCallback);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, response);
res = curl_easy_perform(curl);
if(res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
}
curl_easy_cleanup(curl);
return 0;
}
How do I fix this? Does this mean I can use std::binded functions in the same function, but can't pass them anywhere?
The problem is that you are trying to pass a complex object through ..., as the compiler already tells you. The probably best solution is to first wrap the std::bind in a std::function object, to avoid having to repeat the complete type:
function<void(string)> writeBodyPass(writeBody);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &writeBodyPass);
You then have to fix the signature (and body) of your callback, as you are now receiving a pointer to the std::function object:
size_t WriteMemoryCallback(char * contents, size_t size, size_t nmemb, function<void(string)> *writer)
{
size_t realsize = size * nmemb;
string data(contents, realsize);
(*writer)(data);
return realsize;
}
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
_getch();
return 0;
}
string contents = "";
I would like to save the result of the curl html content in a string, how do I do this?
It's a silly question but unfortunately, I couldn't find anywhere in the cURL examples for C++
thanks!
You will have to use CURLOPT_WRITEFUNCTION to set a callback for writing. I can't test to compile this right now, but the function should look something close to;
static std::string readBuffer;
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
size_t realsize = size * nmemb;
readBuffer.append(contents, realsize);
return realsize;
}
Then call it by doing;
readBuffer.clear();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
// ...other curl options
res = curl_easy_perform(curl);
After the call, readBuffershould have your contents.
Edit: You can use CURLOPT_WRITEDATA to pass the buffer string instead of making it static. In this case I just made it static for simplicity. A good page to look (besides the linked example above) is here for an explanation of the options.
Edit2: As requested, here's a complete working example without the static string buffer;
#include <iostream>
#include <string>
#include <curl/curl.h>
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
std::string readBuffer;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
std::cout << readBuffer << std::endl;
}
return 0;
}
On my blog I have published a simple wrapper class to perform this task.
Usage example:
#include "HTTPDownloader.hpp"
int main(int argc, char** argv) {
HTTPDownloader downloader;
std::string content = downloader.download("https://stackoverflow.com");
std::cout << content << std::endl;
}
Here's the header file:
/**
* HTTPDownloader.hpp
*
* A simple C++ wrapper for the libcurl easy API.
*
* Written by Uli Köhler (techoverflow.net)
* Published under CC0 1.0 Universal (public domain)
*/
#ifndef HTTPDOWNLOADER_HPP
#define HTTPDOWNLOADER_HPP
#include <string>
/**
* A non-threadsafe simple libcURL-easy based HTTP downloader
*/
class HTTPDownloader {
public:
HTTPDownloader();
~HTTPDownloader();
/**
* Download a file using HTTP GET and store in in a std::string
* #param url The URL to download
* #return The download result
*/
std::string download(const std::string& url);
private:
void* curl;
};
#endif /* HTTPDOWNLOADER_HPP */
Here's the source code:
/**
* HTTPDownloader.cpp
*
* A simple C++ wrapper for the libcurl easy API.
*
* Written by Uli Köhler (techoverflow.net)
* Published under CC0 1.0 Universal (public domain)
*/
#include "HTTPDownloader.hpp"
#include <curl/curl.h>
#include <curl/easy.h>
#include <curl/curlbuild.h>
#include <sstream>
#include <iostream>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
string data((const char*) ptr, (size_t) size * nmemb);
*((stringstream*) stream) << data;
return size * nmemb;
}
HTTPDownloader::HTTPDownloader() {
curl = curl_easy_init();
}
HTTPDownloader::~HTTPDownloader() {
curl_easy_cleanup(curl);
}
string HTTPDownloader::download(const std::string& url) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
/* example.com is redirected, so we tell libcurl to follow redirection */
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); //Prevent "longjmp causes uninitialized stack frame" bug
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "deflate");
std::stringstream out;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out);
/* Perform the request, res will get the return code */
CURLcode res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
return out.str();
}
Using the 'new' C++11 lambda functionality, this can be done in a few lines of code.
#ifndef WIN32 #define __stdcall "" #endif //For compatibility with both Linux and Windows
std::string resultBody { };
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &resultBody);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, static_cast<size_t (__stdcall *)(char*, size_t, size_t, void*)>(
[](char* ptr, size_t size, size_t nmemb, void* resultBody){
*(static_cast<std::string*>(resultBody)) += std::string {ptr, size * nmemb};
return size * nmemb;
}
));
CURLcode curlResult = curl_easy_perform(curl);
std::cout << "RESULT BODY:\n" << resultBody << std::endl;
// Cleanup etc
Note the __stdcall cast is needed to comply to the C calling convention (cURL is a C library)
This might not work right away but should give you an idea:
#include <string>
#include <curl.h>
#include <stdio.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main() {
std::string tempname = "temp";
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
FILE *fp = fopen(tempname.c_str(),"wb");
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
fp = fopen(tempname.c_str(),"rb");
fseek (fp , 0 , SEEK_END);
long lSize = ftell (fp);
rewind(fp);
char *buffer = new char[lSize+1];
fread (buffer, 1, lSize, fp);
buffer[lSize] = 0;
fclose(fp);
std::string content(buffer);
delete [] buffer;
}
}
Came out with useful, yet simple solution, which overloads std::ostream::operator<<
#include <ostream>
#include <curl/curl.h>
size_t curlCbToStream (
char * buffer,
size_t nitems,
size_t size,
std::ostream * sout
)
{
*sout << buffer;
return nitems * size;
}
std::ostream & operator<< (
std::ostream & sout,
CURL * request
)
{
::curl_easy_setopt(request, CURLOPT_WRITEDATA, & sout);
::curl_easy_setopt(request, CURLOPT_WRITEFUNCTION, curlCbToStream);
::curl_easy_perform(request);
return sout;
}
Possible drawback of taken approach could be:
typedef void CURL;
That means it covers all known pointer types.
Based on #JoachimIsaksson answer, here is a more verbose output that handles out-of-memory and has a limit for the maximum output from curl (as CURLOPT_MAXFILESIZE limits only based on header information and not on the actual size transferred ).
#DEFINE MAX_FILE_SIZE = 10485760 //10 MiB
size_t curl_to_string(void *ptr, size_t size, size_t count, void *stream)
{
if(((string*)stream)->size() + (size * count) > MAX_FILE_SIZE)
{
cerr<<endl<<"Could not allocate curl to string, output size (current_size:"<<((string*)stream)->size()<<"bytes + buffer:"<<(size * count) << "bytes) would exceed the MAX_FILE_SIZE ("<<MAX_FILE_SIZE<<"bytes)";
return 0;
}
int retry=0;
while(true)
{
try{
((string*)stream)->append((char*)ptr, 0, size*count);
break;// successful
}catch (const std::bad_alloc&) {
retry++;
if(retry>100)
{
cerr<<endl<<"Could not allocate curl to string, probably not enough memory, aborting after : "<<retry<<" tries at 10s apart";
return 0;
}
cerr<<endl<<"Could not allocate curl to string, probably not enough memory, sleeping 10s, try:"<<retry;
sleep(10);
}
}
return size*count;
}
I use Joachim Isaksson's answer with a modern C++ adaptation of CURLOPT_WRITEFUNCTION.
No nagging by the compiler for C-style casts.
static auto WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) -> size_t {
static_cast<string*>(userdata)->append(ptr, size * nmemb);
return size * nmemb;
}