I am trying to download/(and later upload it again) a json file from a ftp server(provider: bplaced.net) into memory using LibCurl, but im getting an error.
Please don't be to harsh im pretty new to this. Also feel free to improve my code thanks :p
Memory methods:
struct MemoryStruct {
char* memory;
size_t size;
};
static size_t
WriteMemoryCallback(void* contents, size_t size, size_t nmemb, void* userp)
{
size_t realsize = size * nmemb;
struct MemoryStruct* mem = (struct MemoryStruct*)userp;
char* ptr = (char*)realloc(mem->memory, mem->size + realsize + 1);
if (ptr == NULL) {
/* out of memory! */
printf("not enough memory (realloc returned NULL)\n");
return 0;
}
mem->memory = ptr;
memcpy(&(mem->memory[mem->size]), contents, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
return realsize;
}
Download Method:
std::string getFTPFileMem() {
CURL* curl;
CURLcode res;
std::string memory;
struct MemoryStruct chunk;
chunk.memory = (char*)malloc(1); /* will be grown as needed by the realloc above */
chunk.size = 0; /* no data at this point */
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL,
"ftp://user:pass#host/jsonLocation/Json.json");
/* send all data to this function */
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
/* we pass our 'chunk' struct to the callback function */
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&chunk);
/* We activate SSL and we require it for both control and data */
curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL);
/* Switch on full protocol/debug output */
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
res = curl_easy_perform(curl);
memory = chunk.memory;
/* always cleanup */
curl_easy_cleanup(curl);
if (CURLE_OK != res) {
/* we failed */
fprintf(stderr, "curl told us %d\n", res);
}
}
free(chunk.memory);
/* we're done with libcurl, so clean it up */
curl_global_cleanup();
return memory;
}
int main() {
std::cout << getFTPFileMem() << "\n";
}
FTP certificate issue:
Related
I am trying to collect some data from a URL. If I do not define any CURLOPT_WRITEFUNCTION and CURLOPT_WRITEDATA I can obviously see the output on console. Then I tried to write that data to memory by copiying the example code, however userdata argument of my callback function returned NULL and I got following exception on line:
char* ptr = (char*)realloc(mem->memory, mem->size + realsize + 1);
Exception thrown: read access violation.
mem was nullptr.
Am I doing something wrong?
Here is my code:
struct MemoryStruct {
char* memory;
size_t size;
};
//-----------------
// Curl's callback
//-----------------
size_t CurlWrapper::curl_cb(char* data, size_t size, size_t nmemb, void* response)
{
size_t realsize = size * nmemb;
std::cout << "CALLBACK CALLED" << std::endl;
MemoryStruct* mem = (struct MemoryStruct*)response;
char* ptr = (char*)realloc(mem->memory, mem->size + realsize + 1);
if (!ptr) {
/* out of memory! */
printf("not enough memory (realloc returned NULL)\n");
return 0;
}
mem->memory = ptr;
memcpy(&(mem->memory[mem->size]), data, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
return realsize;
}
//--------------------
// Do the curl
//--------------------
void CurlWrapper::curl_api(
const std::string& url,
std::string& str_result)
{
MemoryStruct chunk;
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &CurlWrapper::curl_cb);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&chunk);
// TODO: enable ssh certificate
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false); // true
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, false); // 2
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "zlib");
auto res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK) {
// nothing
std::cout << "curl_easy_perform() failed: " << curl_easy_strerror(res) << std::endl;
}
}
}
libcurl version: 7.82.0
Since libcurl is a C library, it does not know anything about C++ member functions or objects. You can overcome this "limitation" with relative ease using for example a static member function that is passed a pointer to the class.
See this example (from the everything curl book).
// f is the pointer to your object.
static size_t YourClass::func(void *buffer, size_t sz, size_t n, void *f)
{
// Call non-static member function.
static_cast<YourClass*>(f)->nonStaticFunction();
}
// This is how you pass pointer to the static function:
curl_easy_setopt(hcurl, CURLOPT_WRITEFUNCTION, YourClass::func);
curl_easy_setopt(hcurl, CURLOPT_WRITEDATA, this);
NOTE: Reopened question with more details.
So here we have some code
someCode.cpp
(...)
MyStruct sct = performRequest(.... .... ....); //Line 1
std::cout << "Line 2";
std::cout << "Line 3";
(...)
someCode.hpp (for struct MyStruct)
(...)
struct MyStruct {
char** memory;
int* response_code;
};
(...)
performRequest() function
static size_t
WriteMemoryCallback(void* contents, size_t size, size_t nmemb, void* userp)
{
size_t realsize = size * nmemb;
struct MemoryStruct* mem = (struct MemoryStruct*)userp;
char* ptr = (char*) realloc(mem->memory, mem->size + realsize + 1);
if (ptr == NULL) {
/* out of memory! */
printf("not enough memory (realloc returned NULL)\n");
return 0;
}
mem->memory = ptr;
memcpy(&(mem->memory[mem->size]), contents, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
return realsize;
}
MyStruct performRequest(std::string requestType, std::string URL, std::string requestParameters, struct curl_slist* headersList, std::string requestFields) {
CURL* curl_handle = curl_easy_init();
CURLcode curl_code;
struct MemoryStruct chunk;
chunk.memory = (char*) malloc(1); /* will be grown as needed by the realloc above */
chunk.size = 0; /* no data at this point */
curl_easy_setopt(curl_handle, CURLOPT_BUFFERSIZE, 102400L);
curl_easy_setopt(curl_handle, CURLOPT_URL, (URL + requestParameters).c_str());
curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, true);
curl_easy_setopt(curl_handle, CURLOPT_HEADER, false);
if (!requestFields.empty()) {
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, requestFields.c_str());
}
if (headersList != NULL) {
curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, headersList);
}
curl_easy_setopt(curl_handle, CURLOPT_MAXREDIRS, 50L);
curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, requestType.c_str());
curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, false);
curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPALIVE, true);
/* send all data to this function, we pass our 'chunk' struct to the callback function */
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void*)& chunk);
curl_code = curl_easy_perform(curl_handle);
/* check for errors */
if (curl_code != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(curl_code));
}
int response_code;
curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response_code);
/* cleanup curl stuff */
curl_easy_cleanup(curl_handle);
MyStruct res;
res.memory = &chunk.memory;
res.response_code = &response_code;
return res;
}
If we debug someCode.cpp and add two breakpoints in lines 2 and 3...
(...)
MyStruct sct = performRequest(···,···,···); //Line 1
[BREAKPOINT (1) HERE] std::cout << "Line 2";
[BREAKPOINT (2) HERE] std::cout << "Line 3";
(...)
...see the following images about the resulting values by the debugger, for the struct sct.
(which corresponds to Breakpoint 1) :)
(which corresponds to Breakpoint 2) :(
As you can see, the values in sct change to NULL without prior instruction. I don't know why this happens, although I suspect it might be something pointer-related...
So to deal with the object lifetime (sct), I changed performRequest() to this:
RequestResponse* performRequest(···) {
(···)
return new RequestResponse(&chunk.memory,&response_code);
}
but still I have the same problem.
Hope you have any idea and thanks!
Your issue is right here:
MyStruct res;
res.memory = &chunk.memory;
res.response_code = &response_code;
return res;
You're supplying res with the address of two local variables. So the second that function ends and you return the struct with those pointers, they're both invalid. Thus leading to this undefined behavior.
You could allocate new pointers on the heap there and seed them with the values of chunk.memory and response_code. Since they're on the heap, they'll persist until you free the memory, so that'd work.
However, I'd strongly suggest moving to a std::vector<std::string> instead of your char** and move to a raw int instead of int* if possible so you don't even need to worry about UB like this.
I am having some issue getting an image into a dynamic memory buffer with cURL.
The bit of code used is the following:
struct memoryStruct {
char *memory;
size_t size;
};
static void* CURL_realloc(void *ptr, size_t size)
{
/* There might be a realloc() out there that doesn't like reallocing
NULL pointers, so we take care of it here */
if(ptr)
return realloc(ptr, size);
else
return malloc(size);
}
size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
{
size_t realsize = size * nmemb;
struct memoryStruct *mem = (struct memoryStruct *)data;
mem->memory = (char *)CURL_realloc(mem->memory, mem->size + realsize + 1);
if (mem->memory)
{
memcpy(&(mem->memory[mem->size]), ptr, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
}
return realsize;
}
int main()
{
std::string everything = "https://upload.wikimedia.org/wikipedia/commons/1/1e/Stonehenge.jpg";
CURL *curl; // CURL objects
CURLcode res;
memoryStruct buffer; // memory buffer
curl = curl_easy_init(); // init CURL library object/structure
if(curl) {
// set up the write to memory buffer
// (buffer starts off empty)
buffer.memory = NULL;
buffer.size = 0;
// (N.B. check this URL still works in browser in case image has moved)
curl_easy_setopt(curl, CURLOPT_URL, everything);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); // tell us what is happening
// tell libcurl where to write the image (to a dynamic memory buffer)
curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl,CURLOPT_WRITEDATA, (void *) &buffer);
// get the image from the specified URL
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
free(buffer.memory);
}
return 0;
}
As an error output I get the error saying the following:
I am however not sure to understand the error being displayed. My guess is that I should enable IDN support for curl? I am however not sure on how to proceed. Does that mean that I would have to recompile the library with IDN enabled? (If I find how to do it)
Thank you for your help
You're passing an std::string object to curl_easy_setopt, while you should be passing a C string. It's actually trying to resolve god-knows-what.
curl_easy_setopt(curl, CURLOPT_URL, everything.c_str());
I'm trying to upload a file to an http server. I'm getting the 200 OK from the server, but the code below is only transmitting 4 bytes.
size_t myclass::read_callback(void *ptr, size_t size, size_t nmemb, void *userp)
{
handler->read(buffer, buffer_size); // buffer_size is 100000
size_t res = handler->gcount();
if( res == 0 )
return 0;
ptr = buffer; // buffer is array of char, defined in myclass
size = res;
nmemb = sizeof(char);
return 1;
}
void myclass::upload_function(const std::string& url)
{
CURL *curl;
CURLcode res;
std::ifstream if_file;
if_file.open("/path_to_file", std::ios::binary);
handler = &if_file; // handler is defined in myclass
/* In windows, this will init the winsock stuff */
res = curl_global_init(CURL_GLOBAL_DEFAULT);
/* Check for errors */
if(res != CURLE_OK) {
// failure
return;
}
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "hostname");
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, myclass::read_callback);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
struct curl_slist *chunk = NULL;
chunk = curl_slist_append(chunk, "Transfer-Encoding: chunked");
chunk = curl_slist_append(chunk, "Content-Type: application/x-mpegURL");
res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK) {
// failed
}
else
{
double speed_upload, total_time;
curl_easy_getinfo(curl, CURLINFO_SPEED_UPLOAD, &speed_upload);
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time);
fprintf(stderr, "Speed: %0.3f b/sec during %.3f seconds\n",
speed_upload, total_time);
}
/* always cleanup */
curl_easy_cleanup(curl);
}
curl_global_cleanup();
if_file.close();
}
The callback doesn't seem to copy data to the buffer. It just assigns the local pointer, quite without any effect.
The callback looks like to be a C++ method that can't be used like that as a callback in a C API that doesn't know about C++ objects...
I've created a PHP and C# app that gets the users Xbox 360 Profile ID from their gamertag.
I was just wondering how I do a web request and put the page contents of the url into a string in C++.
Thanks.
Here's a down-and-dirty C++ HTTP fetcher I wrote using libCURL some time back.
It takes a URL and returns a string with the contents of a get request. Modify and flavor to taste:
/*
* HttpFetcher.h
*/
#ifndef _http_fetcher_
#define _http_fetcher_
#include <stdlib.h>
#include <string>
class HttpFetcher {
public:
std::string simpleGetRequest(std::string url);
private:
// Private methods to handle getting
struct MemoryStruct {
char *memory;
size_t size;
};
static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data);
};
#endif // ifndef _http_fetcher_
/*
* HttpFetcher.cpp
* Much of this is quite similar to examples on the libcURL site
*/
#include "HttpFetcher.h"
#include "curl.h"
#include "easy.h"
#include <iostream>
size_t HttpFetcher::WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
{
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)data;
mem->memory = (char*)realloc(mem->memory, mem->size + realsize + 1);
if (mem->memory == NULL) {
/* out of memory! */
printf("not enough memory (realloc returned NULL)\n");
exit(EXIT_FAILURE);
}
memcpy(&(mem->memory[mem->size]), ptr, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
return realsize;
}
std::string HttpFetcher::simpleGetRequest(std::string url)
{
std::string toReturn = "";
CURL *curl = NULL;
struct MemoryStruct chunk;
chunk.memory = (char*)malloc(1);
chunk.size = 0;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &WriteMemoryCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&chunk);
// Some requests fail without a user agent.
curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0");
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
toReturn = std::string(chunk.memory);
}
free(chunk.memory);
return toReturn;
}