Get remote file size without content length.
size_t readHeader(char* header, size_t size, size_t nitems, void *userdata) {
// this works I get the file size when Content-length is available
if (startsWith(header, "Content-Length:")) {
std::string header_in_string = replaceAll(header, "Content-Length:", "");
long size = atol(header_in_string.c_str());
*file_size = size;
}
else { // do something else }
return size * nitems;
}
// main function
if (curl) {
fp = fopen(temp_file_name, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_CAINFO, "./ca-bundle.crt");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, false);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, this);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, readHeader);
// writing the downloading content to file
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
This provide the solution in C# via GET
Getting remote file size without content-length but I want to know the solution in C++ with libcurl.
Related
I use libcurl to download file, url may be like this below
url = "https://***.com/**/***/abc"
url = "http://***:8000/**/test.txt?***2484d197c16b2e"
url = "http://***:8000/**/test.txt"
It is troublesome to get its file name, so is there a way to get file name using libcurl when downloading it? My download code is below.
size_t writeData(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
int downloadFile(const char *url, const char *saveFile)
{
CURL *curl;
FILE *saveFd;
CURLcode res;
curl = curl_easy_init();
if (curl) {
saveFd = fopen(saveFile,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeData);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, saveFd);
std::string strurl = string(url);
if (strurl.find("https") != string::npos) {
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, false);
}
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(saveFd);
}
return 0;
}
Use CURLOPT_HEADERFUNCTION to get a callback for each of the HTTP headers. You are looking for the header called Content-Disposition.
ok so basically i want the response headers after i sendt my post request.
this is my send post code
pDataInfo->recvHeadBuff = (char*)VirtualAlloc(NULL, 4096, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);
InitPost(pDataInfo);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 5L);
curl_easy_setopt(curl, CURLOPT_URL, pDataInfo->URL);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, pDataInfo->PostData.c_str());
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, pDataInfo->recvHeadBuff);
curl_easy_perform(curl);
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &pDataInfo->repCode);
k so right here im trying to create a buffer that im going to use for my userdata*
callback func:
static size_t header_callback(char* buffer, size_t size,
size_t nitems, void* userdata)
{
strcpy((char*)userdata, buffer);
std::cout << (char*)userdata; // This prints the headers correctly
return nitems * size;
}
now i thought that my buffer would be filled with the characters from the header buffer but when i try to cout the buffer again from outside this callback function i get nothing.
basically need help with a way to use userdata and buffer to get my variable out of this func, ik its noobxd.
Your CURLOPT_HEADERFUNCTION callback is assuming the provided buffer is null-terminated, but it is not. The documentation even says so:
This function gets called by libcurl as soon as it has received header data. The header callback will be called once for each header and only complete header lines are passed on to the callback. Parsing headers is very easy using this. buffer points to the delivered data, and the size of that data is nitems; size is always 1. Do not assume that the header line is null-terminated!
You are also assuming the full header will fit inside the 4K recvHeadBuff that you are allocating, but that is not guaranteed either, so you are risking a buffer overflow.
Try something more like this instead:
std::string headers;
InitPost(pDataInfo);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 5L);
curl_easy_setopt(curl, CURLOPT_URL, pDataInfo->URL);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, pDataInfo->PostData.c_str());
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers);
curl_easy_perform(curl);
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &pDataInfo->repCode);
// use headers as needed...
DWORD size = (headers.size() + 1) * sizeof(char);
pDataInfo->recvHeadBuff = (char*) VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
if (pDataInfo->recvHeadBuff)
CopyMemory(pDataInfo->recvHeadBuff, headers.c_str(), size);
...
static size_t header_callback(char* buffer, size_t size,
size_t nitems, void* userdata)
{
std::string *headers = (std::string*) userdata;
headers->append(buffer, nitems * size);
return nitems * size;
}
Alternatively, you should change pDataInfo->recvHeadBuff to use the same string type that pDataInfo->PostData is using (I'm assuming std::string) rather than being a char*, and then you can do this instead:
InitPost(pDataInfo);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 5L);
curl_easy_setopt(curl, CURLOPT_URL, pDataInfo->URL);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, pDataInfo->PostData.c_str());
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback);
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &(pDataInfo->recvHeaders));
curl_easy_perform(curl);
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &pDataInfo->repCode);
...
static size_t header_callback(char* buffer, size_t size,
size_t nitems, void* userdata)
{
std::string *headers = (std::string*) userdata;
headers->append(buffer, nitems * size);
return nitems * size;
}
I have a curl script:
curl -k -H "Content-Type: application/txt" -X POST --data-binary "#name_file.txt" -g "https://ya.com/file.txt" > .\out.txt
I can use it via console ant it works good.
I implement this script in visual studio on C++:
auto curl = curl_easy_init();
if (curl)
{
FILE* fp = fopen(path_to_file.c_str(), "wb");
curl_easy_setopt(curl, CURLOPT_URL, "https://ya.com/file.txt");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
struct curl_slist *list = NULL;
list = curl_slist_append(list, "Content-Type: application/txt");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); // - H
std::string data_for_send = data_for_send_arr.c_str();
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data_for_send.size());
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data_for_send.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, callbackfunction);
res = curl_easy_perform(curl);
if (res != CURLE_OK) AfxMessageBox(_T("error"));
curl_easy_cleanup(curl);
curl_global_cleanup();
fclose(fp);
}
size_t callbackfunction(void *ptr, size_t size, size_t nmemb, void* userdata)
{
FILE* stream = (FILE*)userdata;
if (!stream)
{
printf("!!! No stream\n");
return 0;
}
size_t written = fwrite((FILE*)ptr, size, nmemb, stream);
return written;
}
i also i need to tell that at one moment i use async call:
async(func_curl_1, data_1);
async(func_curl_2, data_2);
each function create curl and do request for server at the same time.
but this script do not work. it work one time from seven request
server returned for me empty file or broken file.
i do not know why...
try to add in your settings CURLOPT_VERBOSE
i.e.
/* ask libcurl to show us the verbose output */
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
and investigate output
I tried compiling both the examples on this question: Download file using libcurl in C/C++
Here's one of them:
#include <stdio.h>
#include <curl/curl.h>
int main(void)
{
CURL *curl;
FILE *fp;
CURLcode res;
char *url = "http://stackoverflow.com";
char outfilename[FILENAME_MAX] = "page.html";
curl = curl_easy_init();
if (curl)
{
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, NULL);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
the problem is that this example, when run, immediately returns and I get a blank file. Why? I modified to
if (curl)
{
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, NULL);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
} else {
printf("error\n");
}
but I see no error. I tried compiling in both C++ and C, I get the same result on both.
I had the same issue and I fixed it by:
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
According to https://curl.se/libcurl/c/CURLOPT_FOLLOWLOCATION.html, true tells the library to follow HTTP location.
I'm working upon a program in C++, where I have to download a file using cURL, and further open it, but the problem is that when I try to open the file after downloading, it isn't opening. I'm trying to open an .exe file. Here's the portion of code, which is responsible for file download
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename.c_str(), "w");
curl_easy_setopt(curl, CURLOPT_URL, links[index]);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
and the line which should open the downloaded file :
ShellExecute(NULL, "open", fileToLaunch.c_str() , NULL, NULL, SW_HIDE);
When I try to launch manually the file (by clicking on it), Windows returns me an error message which says that respective app isn't a Win32 app. I'm using Visual Studio 2017.
Here's the whole chunk of code :
#include <stdio.h>
#include <curl/curl.h>
#include <curl/easy.h>
#include <string>
#include <iostream>
using namespace std;
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main(void) {
CURL *curl;
FILE *fp;
CURLcode res;
string url = "Here goes the url for file download";
string outfilename = "C:\\1.exe";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename.c_str(), "wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
ShellExecute(NULL, "open", outfilename.c_str(), NULL, NULL, SW_HIDE);
}
Change
fp = fopen(outfilename.c_str(), "w");
to
fp = fopen(outfilename.c_str(), "wb");
You're writing it to disk as text with newline translations by default. You need to write it as binary.
See https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/fopen-wfopen?view=vs-2017 for a fuller explanation.
First of all this line should be deleted :
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
because write_data function has the same functionality, as cURL's CURLOPT_WRITEDATA, and so, the line below the one which should be deleted will be enough, and secondly, on the line where curl_easy_setopt() is firstly met:
curl_easy_setopt(curl, CURLOPT_URL, url);
you should add to the url parameter .c_str(), so it will look like this :
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
because this function is unable to operate with string type of data...
And as Rob K mentioned, you have to change 'w' to 'wb' in fopen() function, because you are working with a binary stream of data.