Related
I have project with curl but size my file 720KB. This is a lot for me. I am use curl only for smtp, how i can reduce my file. I tried use defines discribed in CURL-DISABLE.md but that not worked, i think i configured them incorrectly. Can you give me instruction how i can reduce curl? My code of program.
#include <iostream>
#include <windows.h>
#define CURL_STATICLIB
#include "curl/curl.h"
#pragma comment(lib, "libcurl_a.lib")
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
std::string readBuffer;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "https://google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
std::cout << readBuffer << std::endl;
}
return 0;
}
It work but a lot of size. SCREEN_OF_FILE_IN_PATH
Im trying to find a way to download a file and have the program wait for the download to finish then move onto the next task which in this case would be checking to see if the file exist
#include <stdio.h>
#include <curl/curl.h>
#include <curl/easy.h>
#include <string>
#include <iostream>
#include <stdlib.h>
#include <fstream>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
using namespace std;
int main(void) {
CURL *curl;
FILE *fp;
CURLcode res;
char *url = "http://199.91.153.180/y1plsynnb11g/e9g9mcd3fi5txov/UHC.jar";
char outfilename[FILENAME_MAX] = "UHC.jar";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_URL, "http://199.91.153.180/y1plsynnb11g/e9g9mcd3fi5txov/UHC.jar");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
fclose(fp);
}
ofstream launch;
launch.open("UHC.jar");
if (launch.is_open()){
cout<< "File Downloaded";
}
else {
cout << "Download Failed";
}
}
The HTTP request may have failed. Try adding error checking before /* always cleanup */
if (res)
cout << curl_easy_strerror(res) << endl;
I realized the return value of write_data is wrong. But it should have caused an error...
Anyway, please try:
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written * size;
}
I just realized what was wrong when it to check for the file existing using ofstream it was trying to write to it so it would make the file empty. Just changing it from ofsteam to fstream it works perfectly.
In my program, I want to download a few files. So I took cURL and used this code (taken and modified a little bit from here Download file using libcurl in C/C++):
#include "curl.h"
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int DlZip(){
CURL *curl;
FILE *fp;
CURLcode res;
string url = "http://wordpress.org/extend/plugins/about/readme.txt";
char outfilename[FILENAME_MAX] = "/Users/Me/Desktop/bbb.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
But nothing happened and there weren't any file on my desktop :-/
What is the problem with my code?
Or if you have a simple function to use, could you give me ?
Thanks!
Check Below code
#include <cstdio>
#include <curl/curl.h>
#include <curl/types.h>
#include <curl/easy.h>
#include <string>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main(){
CURL *curl;
FILE *fp;
CURLcode res;
string url = "http://www.joes-hardware.com/tools.html";
char outfilename[FILENAME_MAX] = "./MyText.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
It works fine.
The problem is - this code is not able to use https and hence when the url provided by you when opened in browser and through above code - produce separate responses.
I have used the following code to get to download all the files from the FTP Server
Steps followed are:
1. Creating a FTP list of File
getFTPList(string sHost, string sUser, string sPass, string sUri)
{
CURL *curl;
CURLcode res;
FILE *ftplister;
string host = "ftp://";
host += sHost;
host += "/sample/";
string furl = host + sUri;
string usrpwd = sUser;
usrpwd += ":";
usrpwd += sPass;
/* local file name to store the file as */
ftplister = fopen("ftp-list", "wb"); /* b is binary, needed on win32 */
curl = curl_easy_init();
if(curl) {
/* Get a file listing from sunet */
curl_easy_setopt(curl, CURLOPT_URL, furl.c_str() );
curl_easy_setopt(curl, CURLOPT_USERPWD, usrpwd.c_str());
curl_easy_setopt(curl, CURLOPT_FTPLISTONLY, TRUE);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_list);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, ftplister);
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
/* always cleanup */
curl_easy_cleanup(curl);
}
fclose(ftplister); /* close the local file */
}
Use this list to download the files calling the download functions recursively
int main(){
FILE *ftpfile;
string line;
ftpfile = fopen("ftp-list", "r");
ifstream infile("ftp-list");
while ( getline(infile, line) )
{
string url, ofname, surl = "ftp://myhost/uploader/", sfname = "C:\\CNAP\\";
url = surl + line;
ofname = sfname +line;
cout<<url<<" "<<ofname<<endl;
char* theVal ;
char* theStr ;
theVal = new char [url.size()+1];
theStr = new char [ofname.size()+1];
strcpy(theVal, url.c_str());
strcpy(theStr, ofname.c_str());
downloadFile(theVal, theStr);
}
return 0;
}
Now the download function:
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
void downloadFile(const char* url, const char* ofname)
{
CURL *curl;
FILE *fp;
CURLcode res;
curl = curl_easy_init();
if (curl){
fp = fopen(ofname,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_USERPWD, "user:pass");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
}
It works well when implemented but only to download text files or some files with texts, if I download an image or a docx or a a zip or rar or for that matter any file which is not text it fails, it fails to open after downloading (says invalid file).
I am not sure what am I missing, any help would be appreciated.
I know this is an inefficient way of coding, but I just need the downloads to be right (any file). Working on the efficiency is my next agenda.
PS: Used this method used here
Downloading multiple files with libcurl in C++
Thank you
The file when downloading needs to be opened as a binary file
fp = fopen(ofname,"wb");
Thank you for the help Rob
This is a guess.:
Try clearing the options CURLOPT_TRANSFERTEXT and CURLOPT_CRLF.
Refernce:
http://curl.haxx.se/libcurl/c/curl_easy_setopt.html
#include <stdio.h>
#include <curl/curl.h>
#include <curl/easy.h>
#include <string>
#include <iostream>
#include <fstream>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream);
void getFileList(
const string &strHost,
const string &strUri,
const string &strUser,
const string &strPassWord,
const string &strTargetFile
);
void downloadAllFiles(
const string &strFtpListFile,
const string &strHost,
const string &strUri,
const string &strUser,
const string &strPassWord,
const string &strSaveTargetFolder
);
int main(void)
{
string strHost = "ftp://192.168.0.1:22";
string strUri = "/cobus/test/";
string strUser = "cobus";
string strPassWord = "password";
string strTargetFile = "c:\\cobus\\ftpList.txt";
string strSaveDestFolder = "c:\\cobus\\";
getFileList(strHost, strUri, strUser, strPassWord, strTargetFile);
downloadAllFiles(
strTargetFile,
strHost,
strUri,
strUser,
strPassWord,
strSaveDestFolder
);
}
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream)
{
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
void getFileList(const string &strHost, const string &strUri,
const string &strUser, const string &strPassWord, const string &strTargetFile)
{
CURL *curl;
CURLcode res;
FILE *ftplister;
string strSourceFullUri = strHost + strUri;
string strUserPwInfo = strUser + ":" + strPassWord;
/* local file name to store the file as */
ftplister = fopen(strTargetFile.c_str(), "wb"); /* b is binary, needed on win32 */
curl = curl_easy_init();
if(curl)
{
/* Get a file listing from sunet */
curl_easy_setopt(curl, CURLOPT_URL, strSourceFullUri.c_str() );
curl_easy_setopt(curl, CURLOPT_USERPWD, strUserPwInfo.c_str());
curl_easy_setopt(curl, CURLOPT_FTPLISTONLY, TRUE);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, ftplister);
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
/* always cleanup */
curl_easy_cleanup(curl);
}
fclose(ftplister); /* close the local file */
}
void downloadAllFiles(const string &strFtpListFile, const string &strHost,
const string &strUri, const string &strUser, const string &strPassWord,
const string &strSaveDestFolder)
{
FILE *ftpFile = NULL;
string strFileName = "";
ifstream infile(strFtpListFile.c_str());
if(!infile.is_open())
{
cerr << "can not open ftpList.txt" << endl;
return ;
}
/* ftpList.txt get data, line by line, processing */
while(getline(infile, strFileName))
{
CURL *curl;
FILE *destFilePath;
CURLcode res;
curl = curl_easy_init();
if (curl)
{
destFilePath = fopen((strSaveDestFolder+strFileName).c_str(),"wb");
curl_easy_setopt(curl, CURLOPT_URL, (strHost+strUri+strFileName).c_str());
curl_easy_setopt(curl, CURLOPT_USERPWD, (strUser+":"+strPassWord).c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, destFilePath);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(destFilePath);
}
}// end while
}
I expressed as much as possible to see at once the contents of the Vivian Lobo.
I tried to test to make statements Vivian Lobo. It is good working.
stackOverFlow is hard to write code.
And how can I comment to Vivian lobo article?
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
_getch();
return 0;
}
string contents = "";
I would like to save the result of the curl html content in a string, how do I do this?
It's a silly question but unfortunately, I couldn't find anywhere in the cURL examples for C++
thanks!
You will have to use CURLOPT_WRITEFUNCTION to set a callback for writing. I can't test to compile this right now, but the function should look something close to;
static std::string readBuffer;
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
size_t realsize = size * nmemb;
readBuffer.append(contents, realsize);
return realsize;
}
Then call it by doing;
readBuffer.clear();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
// ...other curl options
res = curl_easy_perform(curl);
After the call, readBuffershould have your contents.
Edit: You can use CURLOPT_WRITEDATA to pass the buffer string instead of making it static. In this case I just made it static for simplicity. A good page to look (besides the linked example above) is here for an explanation of the options.
Edit2: As requested, here's a complete working example without the static string buffer;
#include <iostream>
#include <string>
#include <curl/curl.h>
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
std::string readBuffer;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
std::cout << readBuffer << std::endl;
}
return 0;
}
On my blog I have published a simple wrapper class to perform this task.
Usage example:
#include "HTTPDownloader.hpp"
int main(int argc, char** argv) {
HTTPDownloader downloader;
std::string content = downloader.download("https://stackoverflow.com");
std::cout << content << std::endl;
}
Here's the header file:
/**
* HTTPDownloader.hpp
*
* A simple C++ wrapper for the libcurl easy API.
*
* Written by Uli Köhler (techoverflow.net)
* Published under CC0 1.0 Universal (public domain)
*/
#ifndef HTTPDOWNLOADER_HPP
#define HTTPDOWNLOADER_HPP
#include <string>
/**
* A non-threadsafe simple libcURL-easy based HTTP downloader
*/
class HTTPDownloader {
public:
HTTPDownloader();
~HTTPDownloader();
/**
* Download a file using HTTP GET and store in in a std::string
* #param url The URL to download
* #return The download result
*/
std::string download(const std::string& url);
private:
void* curl;
};
#endif /* HTTPDOWNLOADER_HPP */
Here's the source code:
/**
* HTTPDownloader.cpp
*
* A simple C++ wrapper for the libcurl easy API.
*
* Written by Uli Köhler (techoverflow.net)
* Published under CC0 1.0 Universal (public domain)
*/
#include "HTTPDownloader.hpp"
#include <curl/curl.h>
#include <curl/easy.h>
#include <curl/curlbuild.h>
#include <sstream>
#include <iostream>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
string data((const char*) ptr, (size_t) size * nmemb);
*((stringstream*) stream) << data;
return size * nmemb;
}
HTTPDownloader::HTTPDownloader() {
curl = curl_easy_init();
}
HTTPDownloader::~HTTPDownloader() {
curl_easy_cleanup(curl);
}
string HTTPDownloader::download(const std::string& url) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
/* example.com is redirected, so we tell libcurl to follow redirection */
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); //Prevent "longjmp causes uninitialized stack frame" bug
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "deflate");
std::stringstream out;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &out);
/* Perform the request, res will get the return code */
CURLcode res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
return out.str();
}
Using the 'new' C++11 lambda functionality, this can be done in a few lines of code.
#ifndef WIN32 #define __stdcall "" #endif //For compatibility with both Linux and Windows
std::string resultBody { };
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &resultBody);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, static_cast<size_t (__stdcall *)(char*, size_t, size_t, void*)>(
[](char* ptr, size_t size, size_t nmemb, void* resultBody){
*(static_cast<std::string*>(resultBody)) += std::string {ptr, size * nmemb};
return size * nmemb;
}
));
CURLcode curlResult = curl_easy_perform(curl);
std::cout << "RESULT BODY:\n" << resultBody << std::endl;
// Cleanup etc
Note the __stdcall cast is needed to comply to the C calling convention (cURL is a C library)
This might not work right away but should give you an idea:
#include <string>
#include <curl.h>
#include <stdio.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main() {
std::string tempname = "temp";
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
FILE *fp = fopen(tempname.c_str(),"wb");
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
fp = fopen(tempname.c_str(),"rb");
fseek (fp , 0 , SEEK_END);
long lSize = ftell (fp);
rewind(fp);
char *buffer = new char[lSize+1];
fread (buffer, 1, lSize, fp);
buffer[lSize] = 0;
fclose(fp);
std::string content(buffer);
delete [] buffer;
}
}
Came out with useful, yet simple solution, which overloads std::ostream::operator<<
#include <ostream>
#include <curl/curl.h>
size_t curlCbToStream (
char * buffer,
size_t nitems,
size_t size,
std::ostream * sout
)
{
*sout << buffer;
return nitems * size;
}
std::ostream & operator<< (
std::ostream & sout,
CURL * request
)
{
::curl_easy_setopt(request, CURLOPT_WRITEDATA, & sout);
::curl_easy_setopt(request, CURLOPT_WRITEFUNCTION, curlCbToStream);
::curl_easy_perform(request);
return sout;
}
Possible drawback of taken approach could be:
typedef void CURL;
That means it covers all known pointer types.
Based on #JoachimIsaksson answer, here is a more verbose output that handles out-of-memory and has a limit for the maximum output from curl (as CURLOPT_MAXFILESIZE limits only based on header information and not on the actual size transferred ).
#DEFINE MAX_FILE_SIZE = 10485760 //10 MiB
size_t curl_to_string(void *ptr, size_t size, size_t count, void *stream)
{
if(((string*)stream)->size() + (size * count) > MAX_FILE_SIZE)
{
cerr<<endl<<"Could not allocate curl to string, output size (current_size:"<<((string*)stream)->size()<<"bytes + buffer:"<<(size * count) << "bytes) would exceed the MAX_FILE_SIZE ("<<MAX_FILE_SIZE<<"bytes)";
return 0;
}
int retry=0;
while(true)
{
try{
((string*)stream)->append((char*)ptr, 0, size*count);
break;// successful
}catch (const std::bad_alloc&) {
retry++;
if(retry>100)
{
cerr<<endl<<"Could not allocate curl to string, probably not enough memory, aborting after : "<<retry<<" tries at 10s apart";
return 0;
}
cerr<<endl<<"Could not allocate curl to string, probably not enough memory, sleeping 10s, try:"<<retry;
sleep(10);
}
}
return size*count;
}
I use Joachim Isaksson's answer with a modern C++ adaptation of CURLOPT_WRITEFUNCTION.
No nagging by the compiler for C-style casts.
static auto WriteCallback(char* ptr, size_t size, size_t nmemb, void* userdata) -> size_t {
static_cast<string*>(userdata)->append(ptr, size * nmemb);
return size * nmemb;
}