I'm doing a file download with libcurl in my c++ program. How can i detect if the request is a 404, and not do the file write? The code is:
void GameImage::DownloadImage(string file_name) {
string game_name;
game_name = file_name.substr(file_name.find_last_of("/")+1);
CURL *curl;
FILE *fp;
CURLcode res;
string url = "http://site/"+game_name+".png";
string outfilename = file_name+".png";
cout<<"INFO; attempting to download "<<url<<"..."<<endl;
curl = curl_easy_init();
if (curl) {
cout<<"INFO; downloading "<<url<<"..."<<endl;
fp = fopen(outfilename.c_str(), "wb");
cout<<"INFO; trying to open "<<outfilename<<" for file output"<<endl;
if (fp != NULL) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GameImage::WriteData);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
res = curl_easy_perform(curl);
long http_code = 0;
curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code);
curl_easy_cleanup(curl);
fclose(fp);
}
else {
cout<<"GameImage::DownloadImage; Couldn't open output file"<<endl;
}
}
}
size_t GameImage::WriteData(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
I can delete the 404 response after the transfer occurs, but it would be good to not even save the response.
You can check against CURLE_HTTP_RETURNED_ERROR
This is returned if CURLOPT_FAILONERROR is set to true and the HTTP server returns an error code that is >= 400. You can't grab the specific HTTP response code, but should be enough to accomplish what you want.
I know this is an old post, but the error you're doing is that you're not checking the return value of curl_easy_perform. Setting CURLOPT_FAILONERROR will not crash the program, instead, it will notify you of the error through the return variable you named res. To get rid of the empty file, you could do something like this:
void GameImage::DownloadImage(string file_name) {
string game_name;
game_name = file_name.substr(file_name.find_last_of("/")+1);
CURL *curl;
FILE *fp;
CURLcode res;
string url = "http://site/"+game_name+".png";
string outfilename = file_name+".png";
cout<<"INFO; attempting to download "<<url<<"..."<<endl;
curl = curl_easy_init();
if (curl) {
cout<<"INFO; downloading "<<url<<"..."<<endl;
fp = fopen(outfilename.c_str(), "wb");
cout<<"INFO; trying to open "<<outfilename<<" for file output"<<endl;
if (fp == NULL) {
cout<<"GameImage::DownloadImage; Couldn't open output file"<<endl;
curl_easy_cleanup(curl);
return;
}
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GameImage::WriteData);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
res = curl_easy_perform(curl);
fclose(fp);
if (res != CURLE_OK) {
cout<<"GameImage::DownloadImage; Failed to download file"<<endl;
remove(outfilename.c_str());
}
curl_easy_cleanup(curl);
}
}
size_t GameImage::WriteData(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
Related
I am making a web crawler
and I have the following code but the problem is that it also downloads binary data and I don't want that to happen. How do I prevent it
size_t HTML::WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) {
size_t realsize = size * nmemb;
if(contents!=NULL||userp!=NULL){
std::string* str=(std::string*)userp;
str->reserve(realsize);
auto c_str=(char*)contents;
if(c_str!=NULL){
for(size_t i=0;i<realsize;i++){
str->push_back(c_str[i]);
}
}
}
return realsize;
}
HTML_CODE HTML::get_html(std::string url) {
std::string chunk;
CURL *curl_handle=curl_easy_init();
CURLcode res;
if(curl_handle) {
curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk);
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, USER_AGENT);
res = curl_easy_perform(curl_handle);
if(res != CURLE_OK) {
std::cout<<"Can't get html content from "<<url<<"\n";
fprintf(stderr, "error: %s\n", curl_easy_strerror(res));
return {"",""};
}
curl_easy_cleanup(curl_handle);
}
else{
std::cout<<"Error: Couldn't create a curl instance"<<std::endl;
return {"",""};
}
return {.url=url,.content=chunk};
}
Things I have tried:-
Check if the data has a null terminator
Check if the char is an assci letter(it wont work with other language)
I am trying to download a .txt file from a server which I can access via the web browser on my raspberry pi.
Curl library gives segmentation error when I am trying to do this. Here is the code I am using.
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
int checkNewFiles(){
CURL *curl;
FILE *fp;
CURLcode res;
string url = "http://52.233.176.151:1880/files/device/software/text.txt";
char outfilename[FILENAME_MAX] = "/home/pi/Desktop/project/cpp/ab.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
I found the problem, what is url.c_str() doing?
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
change this to
curl_easy_setopt(curl, CURLOPT_URL, url);
Example : Curl program that download the text file.
Offcourse you need to add this neccessary header file here.
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main(void) {
CURL *curl;
FILE *fp;
CURLcode res;
const char *url = "http://localhost/yourfile.txt";
char outfilename[FILENAME_MAX] = "C:\\outfile.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); /* enable failure on http errors */
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
if(res != CURLE_OK) { /* check that the operation was successful */
printf("curl_easy_perform(): %s\n", curl_easy_strerror(res));
}
/* always cleanup */
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
I noticed you're not checking for errors after fopen. If it fails, it returns a NULL pointer, which would cause a segfault when curl attempts to write to it.
I'm not convinced that c_str() was the culprit to your segfault in the original question as I have used that in numerous applications with no problems.
I'm trying to upload a file to an http server. I'm getting the 200 OK from the server, but the code below is only transmitting 4 bytes.
size_t myclass::read_callback(void *ptr, size_t size, size_t nmemb, void *userp)
{
handler->read(buffer, buffer_size); // buffer_size is 100000
size_t res = handler->gcount();
if( res == 0 )
return 0;
ptr = buffer; // buffer is array of char, defined in myclass
size = res;
nmemb = sizeof(char);
return 1;
}
void myclass::upload_function(const std::string& url)
{
CURL *curl;
CURLcode res;
std::ifstream if_file;
if_file.open("/path_to_file", std::ios::binary);
handler = &if_file; // handler is defined in myclass
/* In windows, this will init the winsock stuff */
res = curl_global_init(CURL_GLOBAL_DEFAULT);
/* Check for errors */
if(res != CURLE_OK) {
// failure
return;
}
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "hostname");
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, myclass::read_callback);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
struct curl_slist *chunk = NULL;
chunk = curl_slist_append(chunk, "Transfer-Encoding: chunked");
chunk = curl_slist_append(chunk, "Content-Type: application/x-mpegURL");
res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK) {
// failed
}
else
{
double speed_upload, total_time;
curl_easy_getinfo(curl, CURLINFO_SPEED_UPLOAD, &speed_upload);
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time);
fprintf(stderr, "Speed: %0.3f b/sec during %.3f seconds\n",
speed_upload, total_time);
}
/* always cleanup */
curl_easy_cleanup(curl);
}
curl_global_cleanup();
if_file.close();
}
The callback doesn't seem to copy data to the buffer. It just assigns the local pointer, quite without any effect.
The callback looks like to be a C++ method that can't be used like that as a callback in a C API that doesn't know about C++ objects...
In my program, I want to download a few files. So I took cURL and used this code (taken and modified a little bit from here Download file using libcurl in C/C++):
#include "curl.h"
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int DlZip(){
CURL *curl;
FILE *fp;
CURLcode res;
string url = "http://wordpress.org/extend/plugins/about/readme.txt";
char outfilename[FILENAME_MAX] = "/Users/Me/Desktop/bbb.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
But nothing happened and there weren't any file on my desktop :-/
What is the problem with my code?
Or if you have a simple function to use, could you give me ?
Thanks!
Check Below code
#include <cstdio>
#include <curl/curl.h>
#include <curl/types.h>
#include <curl/easy.h>
#include <string>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main(){
CURL *curl;
FILE *fp;
CURLcode res;
string url = "http://www.joes-hardware.com/tools.html";
char outfilename[FILENAME_MAX] = "./MyText.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
It works fine.
The problem is - this code is not able to use https and hence when the url provided by you when opened in browser and through above code - produce separate responses.
I have used the following code to get to download all the files from the FTP Server
Steps followed are:
1. Creating a FTP list of File
getFTPList(string sHost, string sUser, string sPass, string sUri)
{
CURL *curl;
CURLcode res;
FILE *ftplister;
string host = "ftp://";
host += sHost;
host += "/sample/";
string furl = host + sUri;
string usrpwd = sUser;
usrpwd += ":";
usrpwd += sPass;
/* local file name to store the file as */
ftplister = fopen("ftp-list", "wb"); /* b is binary, needed on win32 */
curl = curl_easy_init();
if(curl) {
/* Get a file listing from sunet */
curl_easy_setopt(curl, CURLOPT_URL, furl.c_str() );
curl_easy_setopt(curl, CURLOPT_USERPWD, usrpwd.c_str());
curl_easy_setopt(curl, CURLOPT_FTPLISTONLY, TRUE);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_list);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, ftplister);
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
/* always cleanup */
curl_easy_cleanup(curl);
}
fclose(ftplister); /* close the local file */
}
Use this list to download the files calling the download functions recursively
int main(){
FILE *ftpfile;
string line;
ftpfile = fopen("ftp-list", "r");
ifstream infile("ftp-list");
while ( getline(infile, line) )
{
string url, ofname, surl = "ftp://myhost/uploader/", sfname = "C:\\CNAP\\";
url = surl + line;
ofname = sfname +line;
cout<<url<<" "<<ofname<<endl;
char* theVal ;
char* theStr ;
theVal = new char [url.size()+1];
theStr = new char [ofname.size()+1];
strcpy(theVal, url.c_str());
strcpy(theStr, ofname.c_str());
downloadFile(theVal, theStr);
}
return 0;
}
Now the download function:
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
void downloadFile(const char* url, const char* ofname)
{
CURL *curl;
FILE *fp;
CURLcode res;
curl = curl_easy_init();
if (curl){
fp = fopen(ofname,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_USERPWD, "user:pass");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
}
It works well when implemented but only to download text files or some files with texts, if I download an image or a docx or a a zip or rar or for that matter any file which is not text it fails, it fails to open after downloading (says invalid file).
I am not sure what am I missing, any help would be appreciated.
I know this is an inefficient way of coding, but I just need the downloads to be right (any file). Working on the efficiency is my next agenda.
PS: Used this method used here
Downloading multiple files with libcurl in C++
Thank you
The file when downloading needs to be opened as a binary file
fp = fopen(ofname,"wb");
Thank you for the help Rob
This is a guess.:
Try clearing the options CURLOPT_TRANSFERTEXT and CURLOPT_CRLF.
Refernce:
http://curl.haxx.se/libcurl/c/curl_easy_setopt.html
#include <stdio.h>
#include <curl/curl.h>
#include <curl/easy.h>
#include <string>
#include <iostream>
#include <fstream>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream);
void getFileList(
const string &strHost,
const string &strUri,
const string &strUser,
const string &strPassWord,
const string &strTargetFile
);
void downloadAllFiles(
const string &strFtpListFile,
const string &strHost,
const string &strUri,
const string &strUser,
const string &strPassWord,
const string &strSaveTargetFolder
);
int main(void)
{
string strHost = "ftp://192.168.0.1:22";
string strUri = "/cobus/test/";
string strUser = "cobus";
string strPassWord = "password";
string strTargetFile = "c:\\cobus\\ftpList.txt";
string strSaveDestFolder = "c:\\cobus\\";
getFileList(strHost, strUri, strUser, strPassWord, strTargetFile);
downloadAllFiles(
strTargetFile,
strHost,
strUri,
strUser,
strPassWord,
strSaveDestFolder
);
}
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream)
{
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
void getFileList(const string &strHost, const string &strUri,
const string &strUser, const string &strPassWord, const string &strTargetFile)
{
CURL *curl;
CURLcode res;
FILE *ftplister;
string strSourceFullUri = strHost + strUri;
string strUserPwInfo = strUser + ":" + strPassWord;
/* local file name to store the file as */
ftplister = fopen(strTargetFile.c_str(), "wb"); /* b is binary, needed on win32 */
curl = curl_easy_init();
if(curl)
{
/* Get a file listing from sunet */
curl_easy_setopt(curl, CURLOPT_URL, strSourceFullUri.c_str() );
curl_easy_setopt(curl, CURLOPT_USERPWD, strUserPwInfo.c_str());
curl_easy_setopt(curl, CURLOPT_FTPLISTONLY, TRUE);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, ftplister);
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
/* always cleanup */
curl_easy_cleanup(curl);
}
fclose(ftplister); /* close the local file */
}
void downloadAllFiles(const string &strFtpListFile, const string &strHost,
const string &strUri, const string &strUser, const string &strPassWord,
const string &strSaveDestFolder)
{
FILE *ftpFile = NULL;
string strFileName = "";
ifstream infile(strFtpListFile.c_str());
if(!infile.is_open())
{
cerr << "can not open ftpList.txt" << endl;
return ;
}
/* ftpList.txt get data, line by line, processing */
while(getline(infile, strFileName))
{
CURL *curl;
FILE *destFilePath;
CURLcode res;
curl = curl_easy_init();
if (curl)
{
destFilePath = fopen((strSaveDestFolder+strFileName).c_str(),"wb");
curl_easy_setopt(curl, CURLOPT_URL, (strHost+strUri+strFileName).c_str());
curl_easy_setopt(curl, CURLOPT_USERPWD, (strUser+":"+strPassWord).c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, destFilePath);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(destFilePath);
}
}// end while
}
I expressed as much as possible to see at once the contents of the Vivian Lobo.
I tried to test to make statements Vivian Lobo. It is good working.
stackOverFlow is hard to write code.
And how can I comment to Vivian lobo article?