Why this is not working ? libcurl & c++ - c++

I am new to libcurl and am trying to get filelist from a server using ftp, going through the examples and some other posts here, I have come up with the following code. But when I run it, it returns error message :
Failed writing body (4294967295 != 129)
in the error string set by CURLOPT_ERRORBUFFER. The curl_easy_strerror( res ) returns:
Failed writing received data to disk/application
struct FtpFile
{
const char *filename;
FILE *stream;
};
static size_t fileWrite( void *buffer, size_t size, size_t nmemb, void *stream )
{
struct FtpFile *out=(struct FtpFile *)stream;
if(out && !out->stream)
{
out->stream=fopen(out->filename, "wb");
if(!out->stream)
{
cout << out->filename << " open failure [fileWrite] " << endl;
return -1;
}
}
size_t written = fwrite(buffer, size, nmemb, out->stream);
cout << written << endl;
if(written <= 0)
cout << "Nothing written : " << written;
return written;
}
void getFileList( const char* url, const char* fname )
{
CURL *curl;
CURLcode res;
FILE *ftpfile;
const char *errmsg;
ftpfile = fopen(fname, "wb");
if ( ftpfile == NULL )
{
cout << fname << " open failure [getFileList] " << endl ;
return;
}
curl = curl_easy_init();
if(curl)
{
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fileWrite);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, ftpfile);
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errmsg);
curl_easy_setopt(curl, CURLOPT_USERNAME, "username");
curl_easy_setopt(curl, CURLOPT_PASSWORD, "password");
curl_easy_setopt(curl, CURLOPT_FTPLISTONLY,TRUE);
res = curl_easy_perform(curl);
if(res != CURLE_OK)
{
fprintf( stderr, "curl_easy_perform() failed: %s\nError Message: %s\n", curl_easy_strerror( res ), errmsg );
}
curl_easy_cleanup(curl);
}
fclose(ftpfile);
}
int main(int argc, char *argv[])
{
getFileList( "ftp://ftp.example.com/public/somefolder/", "file-list.txt" );
system("PAUSE");
return EXIT_SUCCESS;
}

4294967295 is the 32 bit unsigned version of -1, which probably is what your callback returned and thus libcurl considered it an error and stopped everything.
Where is ->filename assigned?
You pass in a FILE * to CURLOPT_WRITEDATA and yet your callback reads it as a struct FtpFile *...

First, your fileWrite function should return the number of bytes written. fwrite returns the number of objects written. So you have to return written * size;
Then, as Daniel Stenberg notes, you are passing a FILE * to fileWrite but using it as a struct FtpFile *. Try this one instead:
static size_t fileWrite( void *buffer, size_t size, size_t nmemb, void *stream )
{
size_t written = fwrite(buffer, size, nmemb, (FILE*)stream);
std::cout << written * size << std::endl;
if(written <= 0)
std::cout << "Nothing written : " << written;
return written * size;
}

Related

What may cause a CURLE_UNKNOWN_OPTION in result of curl_easy_getinfo for CURLINFO_RESPONSE_CODE

I'm trying to get a response code for HTTP request inside WriteCallback. I'm sure it worked in my main app just a few days ago, but doesn't work now. However, I tried to checkout previous branch and commits in git, actually they have the same problem now. I cann't give you full reproducible code because it's too large and complex, but I can show you my test app which illustrates what I'm doing:
#include "curl/curl.h"
#include <iostream>
#include <string>
#pragma comment(lib, "Ws2_32.lib")
#pragma comment(lib, "crypt32")
#pragma comment(lib, "normaliz.lib")
#pragma comment(lib, "legacy_stdio_definitions.lib")
namespace details {
FILE _iob[] = { *stdin, *stdout, *stderr };
}
extern "C" FILE* __cdecl __iob_func(void)
{
return details::_iob;
}
using namespace std;
CURL* curl = nullptr;
size_t writeData(void* ptr, size_t size, size_t nmemb, FILE* stream)
{
long responseCode = -1;
auto errorCode = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode); // POINT 1
if (errorCode || (responseCode != 200 && responseCode != 206)) {
cout << "\n writeData FAILED: errorCode: " << errorCode << ", responseCode " << responseCode << '\n';
return 0;
}
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
size_t headerWriter(void* contents, size_t size, size_t nmemb, void* userp)
{
auto header = (std::string((char*)contents, size * nmemb));
cout << "Got header: " << header;
return size * nmemb;
}
int ŅurlProgressCallback(void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
{
return CURL_PROGRESSFUNC_CONTINUE;
}
bool setupCurl(CURL* curl, const std::string& url, FILE* file)
{
static const std::string userAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3202.75 "
"Safari/537.36 MyApp";
bool hasError = (bool)curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
hasError |= (bool)curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
hasError |= (bool)curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
hasError |= (bool)curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
hasError |= (bool)curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
hasError |= (bool)curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5);
hasError |= (bool)curl_easy_setopt(curl, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
hasError |= (bool)curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeData);
hasError |= (bool)curl_easy_setopt(curl, CURLOPT_WRITEDATA, file);
hasError |= (bool)curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerWriter);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, ŅurlProgressCallback);
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, curl);
return hasError;
}
FILE* openFile(const std::string& filePath)
{
auto deleteSuccess = DeleteFileA(filePath.c_str());
if (!deleteSuccess) {
auto error = GetLastError();
if (error != ERROR_FILE_NOT_FOUND) {
cout << "Failed to delete file " << filePath << ", code " << std::to_string(error) << '\n';
return nullptr;
}
}
FILE* file = nullptr;
auto error = fopen_s(&file, filePath.c_str(), "wb");
if (error) {
cout << "Cann't open file " << filePath << '\n';
}
return file;
}
int main()
{
curl_global_init(CURL_GLOBAL_ALL);
curl = curl_easy_init();
if (!curl) {
return -1;
}
const std::string filePath = "C:\\test-curl.txt";
FILE* file = openFile(filePath);
if (!file) {
return -1;
}
const std::string url = "https://curl.se/docs/manpage.html";
bool hasError = setupCurl(curl, url, file);
if (hasError) {
return -1;
}
auto errorCode = curl_easy_perform(curl);
if (errorCode) {
cout << "Curl failed: error code " << errorCode << '\n';
} else {
cout << "\nFinished successfully\n";
}
long responseCode = -1;
errorCode = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode); // POINT 2
if (errorCode) {
cout << "\n failed to get response code\n";
}
fclose(file);
curl_easy_cleanup(curl);
curl_global_cleanup();
return 0;
}
I omitted some checks and proper deinitializing for brevity. This test app works as expected and it uses the same static build of libcurl as my main app. The problem is my main app gets an CURLE_UNKNOWN_OPTION error at point 1:
auto errorCode = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode); // errorCode == CURLE_UNKNOWN_OPTION
The curl_easy_getinfo returns CURLE_OK at point 2 in both apps. I'm sure I didn't change any system settings and I didn't make a libcurl recompilation.
Questions are: should curl_easy_getinfo for CURLINFO_RESPONSE_CODE actually work inside write callback? If yes, what may cause it to fail and return an CURLE_UNKNOWN_OPTION error?
P.S.: I found that in my main app I'm getting CURLE_UNKNOWN_OPTION also in progress callback and write header callback.
P.P.S.: I also found that I'm getting CURLE_UNKNOWN_OPTION error for any info parameter - e.g., I tried CURLINFO_EFFECTIVE_METHOD, CURLINFO_EFFECTIVE_URL and CURLINFO_HTTP_VERSION.
Holy Grail, sorry. I'm feeling so stupid. In my main app I just forgot to update a curl handle after clearing a buffer which I pass to callback as user' structure. So CURLE_UNKNOWN_OPTION in my case actually just meant somethin like "Invalid curl handle".

function returns empty value for std::String

Using refresh to get the output result from as Flask web server, when the function is executed it prints the result needed to the screen but when it is time to print out it does not print the out value and all you get is an empty line
Why is this happening
size_t CurlWrite_CallbackFunc_StdString(void* contents, size_t size, size_t nmemb, std::string* s)
{
size_t newLength = size * nmemb;
try
{
s->append((char*)contents, newLength);
}
catch (std::bad_alloc& e)
{
//handle memory problem
return 0;
}
return newLength;
}
std::string refresh(std::string path) {
CURL* curl;
CURLcode res;
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
const char* path_ = path.c_str();
std::string s;
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, path_);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "out=");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
/* always cleanup */
curl_easy_cleanup(curl);
}
std::cout << s << std::endl;
return s;
}
driver code
int main() {
std::string out = refresh(server_path+"/refresh");
std::cout << out << std::endl;
}

C++ libcurl responses only show first line of response

EDIT: After running a few more commands, it seems that only lines starting with * are being shown. Any way to stop that?
I'm trying to use the libcurl library with my C++ program to recieve emails using the IMAP protocol. The issue is that the response from the CURL calls are only returning one line, and cutting off the rest of what is supposed to be returned. I know that the request itself is working because when I turn on verbose mode with curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);, it has the full result within the output.
For example, when I use curl_easy_setopt(curl, "FETCH 10000 (FLAGS BODY[HEADER.FIELDS (DATE FROM SUBJECT)])"); the response is:
* 10000 FETCH (FLAGS (\Seen) BODY[HEADER.FIELDS (DATE FROM SUBJECT)] {140}
When I turn on verbose mode, the console includes the full result:
< A003 OK [READ-WRITE] INBOX selected. (Success)
> A004 FETCH 10000 (FLAGS BODY[HEADER.FIELDS (DATE FROM SUBJECT)])
< * 10000 FETCH (FLAGS (\Seen) BODY[HEADER.FIELDS (DATE FROM SUBJECT)] {140}
< Date: Wed, 1 Apr 2020 01:59:45 -0500
< From: [REMOVED NAME] <[REMOVED EMAIL]>
< Subject: Testing subject here
<
< )
< A004 OK Success
I have tried multiple ways of outputting the data: to stdout, an external file, and in memory (then printed). I feel like the CURL result just isn't returning more than the first line for some reason. Here is the code if it helps (I'm using wxWidgets):
App.cpp - Ignore most of this, just calls the functions
#include "App.h"
#include "Frame.h"
#include "Config.h"
#include "IMAP.h"
wxIMPLEMENT_APP(App);
bool App::OnInit() {
curl_global_init(CURL_GLOBAL_SSL);
try {
Config config("settings.conf");
for(const auto& account : config["accounts"]) {
const char* url = account["incoming_server"].asCString();
const char* username = account["username"].asCString();
const char* password = account["password"].asCString();
IMAP connection(url, IMAP_DEFAULT_PORT);
if(connection.auth(username, password) == IMAP_SUCCESS) {
connection.select("INBOX");
auto folders = connection.getFolders();
}
}
} catch(std::exception& e) {
std::cerr << e.what() << std::endl;
exit(0);
}
curl_global_cleanup();
exit(1);
Frame *frame = new Frame("MailDuck", wxDefaultPosition, wxSize(100, 100));
frame->Show(true);
frame->Centre();
return true;
}
App.cpp
#include "IMAP.h"
IMAP::IMAP(const char *url, int port) {
this->url = url;
this->port = port;
}
IMAP_CODE IMAP::auth(const char *username, const char *password) {
this->username = username;
this->password = password;
if(execute().size == -1) return IMAP_FAILED;
else return IMAP_SUCCESS;
}
std::vector<Folder> IMAP::getFolders() {
std::vector<Folder> folders;
memory mem = execute("FETCH 10000 (FLAGS BODY[HEADER.FIELDS (DATE FROM SUBJECT)])");
std::cout << mem.response << std::endl;
free(mem.response);
return folders;
}
IMAP::memory IMAP::execute(const char *command) {
CURL *curl;
CURLcode res = CURLE_OK;
struct memory chunk { (char *) malloc(1), 0 };
curl = curl_easy_init();
std::string finalURL;
finalURL.append(this->url);
finalURL.append("/");
finalURL.append(this->mailbox);
if(curl) {
curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL);
curl_easy_setopt(curl, CURLOPT_USERNAME, this->username);
curl_easy_setopt(curl, CURLOPT_PASSWORD, this->password);
curl_easy_setopt(curl, CURLOPT_URL, finalURL.c_str());
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_function);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, command);
res = curl_easy_perform(curl);
if(res != CURLE_OK) chunk.size = -1;
curl_easy_cleanup(curl);
} else chunk.size = -1;
if(chunk.size == -1) std::cerr << "ERROR >> Unable to call command: " << command << std::endl;
return chunk;
}
size_t IMAP::write_function(void *data, size_t size, size_t nmemb, void *userp) {
size_t real_size = size * nmemb;
auto *mem = (struct memory *) userp;
char *ptr = (char *) realloc(mem->response, mem->size + real_size + 1);
if(ptr == NULL) return 0;
mem->response = ptr;
memcpy(&(mem->response[mem->size]), data, real_size);
mem->size += real_size;
mem->response[mem->size] = 0;
return real_size;
}
void IMAP::select(const char *box) {
this->mailbox = box;
}
IMAP.h
#ifndef MAILDUCK_IMAP_H
#define MAILDUCK_IMAP_H
#include <map>
#include <vector>
#include <string>
#include <json/json.h>
#include <curl/curl.h>
#include <iostream>
#include <istream>
enum {
IMAP_DEFAULT_PORT = 993,
};
enum IMAP_CODE {
IMAP_SUCCESS = 1,
IMAP_FAILED = 0
};
class Folder : public std::vector<Folder> {
public:
std::string name;
};
class IMAP {
private:
const char *url, *username, *password, *mailbox = "";
int port;
struct memory { char *response; size_t size; };
memory execute(const char *command = nullptr);
static size_t write_function(void *data, size_t size, size_t nmemb, void *userp);
public:
IMAP(const char *url, int port);
IMAP_CODE auth(const char* username, const char* password);
std::vector<Folder> getFolders();
void select(const char *mailbox);
};
#endif
I've been stuck on this for a bit, so I appreciate any insight anyone could provide.
Figured it out! For all who get stuck like me:
Apparently the rest of the data is stuck in the header data, so add this:
struct memory chunk { (char *) malloc(1), 0 };
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)&chunk);
And then my write function is exactly like the other one...
size_t IMAP::header_callback(char *data, size_t size, size_t nmemb, void *userp) {
size_t real_size = size * nmemb;
auto *mem = (struct memory *) userp;
char *ptr = (char *) realloc(mem->response, mem->size + real_size + 1);
if(ptr == NULL) return 0;
mem->response = ptr;
memcpy(&(mem->response[mem->size]), data, real_size);
mem->size += real_size;
mem->response[mem->size] = 0;
return real_size;
}
Now I'm getting the full response.

Libcurl: image is damaged when using fstream::write instead of fwrite

I'm trying to download image file from a url. I followed the example using fwrite and it succeeded. Now I'm trying to use fstream::write to save the data (ios::binary), but the data is damaged. Here is my code:
#include"stdafx.h"
#include<fstream>
#include<iostream>
#include <curl/curl.h>
#include <string.h>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, char* out) {
//void *ptr, size_t size, size_t nmemb, File* fp
fstream file;
if (file.is_open()){
file.close();
file.clear();
}
file.open(out, ios::out | ios::binary);
if (file.is_open()){
cout << "open successfully\n" << endl;
file.write((char*)ptr, nmemb*size); // Does it correct?
};
// fwrite(ptr,size,nmemb,fp);
file.close();
file.clear();
cout <<"\n sizeof(ptr): " << sizeof(ptr) //size of ptr[0]?
<<"\n sizeof(char): " << sizeof(char)
<<"\n size: " << size
<<"\n nmemb: " << nmemb<< endl;
return size*nmemb;
}
I'm confused about the parameters in write_data.
According to CURLOPT_WRITEFUNCTION
size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata);
"ptr points to the delivered data, and the size of that data is size multiplied with nmemb."
......so what are the meanings of size and nmemb?
When tried to download the data from a website,I printed first 3 parameters. It seems that char*ptr is the memory address that data stored (as 'char a[]'?), and size is element's size, nmemb is number of the elements. So the data size = size * nmemb. Am I correct?
The output is confusing too:
open successfully
sizeof(ptr):4
sizeof(char):1
size:1
nmemb:2715
open successfully
sizeof(ptr):4
sizeof(char):1
size:1
nmemb:4865
download successfully
When download the same url, nmemb and files' open times often change.
I'm also confused about 'sizeof(ptr)', it returns '4'(size of int?). How can I use the 'sizeof' to get the size of data memory, so that I can proof that data size is 'size * nmemb'?
CURLcode download(char* url,char* out){
CURL *curl = NULL;
//FILE *fp = NULL;
CURLcode res;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, out); //fp
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
return res;
}
else
{
return CURLE_FAILED_INIT;
}
}
int main()
{
CURLcode res = download("http://XXXXXX.gif", "D:\\test.gif");
if (CURLE_OK == res)
cout << "download successfully.\n" << endl;
else
cout<<"cannot download.\n"<<endl;
return 0;
}
Thanks! :)
This callback can be multiple times per file. You should NOT create a new file stream every time the function is called - you should pass it in using the user data parameter. Otherwise you will just keep overwriting the data at the beginning of the file.
Here is an example implementation:
size_t write_data(char *ptr, size_t size, size_t nmemb, void *userdata)
{
std::ofstream *out = static_cast<std::ofstream *>(userdata);
size_t nbytes = size * nmemb;
out->write(ptr, nbytes);
return nbytes;
}
You also need to adjust the call to curl_easy_setopt with the parameter CURLOPT_WRITEDATA to actually pass your file stream. Make sure that the stream does not go out of scope while the functions run!
CURLcode download(char* url, char* out) {
CURL *curl = NULL;
std::ofstream output(out, ios::binary);
CURLcode res;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &output);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
return res;
}
else
{
return CURLE_FAILED_INIT;
}
}

c++ how to write a newline into CURLOPT_

I have tried all the obvious methods to write a newline at the end of the file, at the end of the loop from a CURLOPT_ stream.
I do not get an error, but also no newline is ever written.
How to you insert a newline into CURLOPT_ ?
#include <iostream>
#include <stdio.h>
#include <curl/curl.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://pubapi.cryptsy.com/api.php?method=singlemarketdata&marketid=155");
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
FILE * pFile;
pFile = fopen ("/home/coinz/cryptsy/myfile.txt","a+");
if (pFile!=NULL)
{
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, pFile);
res = curl_easy_perform(curl);
std::cout << pFile << std::endl;
//pFile << "\n\r";
fclose (pFile);
}
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}
Try fprintf(pFile, "\n"); instead of the << operator.