I am currently using CURL library, I tried a simple example and I noticed that the heap memory increases every time I make a request. This is a very important problem, especially when you are trying to use multithread.
Does anyone know the problem?
static int Swriter(char *data, size_t size, size_t nmemb, std::string *writerData)
{
if(writerData == NULL)
return 0;
writerData->append(data, size*nmemb);
return size * nmemb;
}
static void RequestReadJson(std::string url, std::string &content)
{
CURL *curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Swriter);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
}
int main(int argc, wchar_t* argv[]) {
curl_global_init(CURL_GLOBAL_DEFAULT);
std::string content;
std::string url("www.google.com");
for(int i=0;i<300;i++)
RequestReadJson(url, content); //Heap increase
curl_global_cleanup();
}
Heap increase
You append the new downloaded content to the old, hence the heap increase:
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
which leads to:
writerData->append(data, size*nmemb);
You'd better return a fresh string:
static std::string RequestReadJson(std::string url)
{
std::string content;
CURL *curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Swriter);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
return content;
}
Related
I've been trying to fetch data from steam community market ,
Code :
#include <iostream>
#include <string>
#include <curl/curl.h>
static size_t WriteCallback(void* contents, size_t size, size_t nmemb, void* userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL* curl;
CURLcode res;
std::string readBuffer;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://steamcommunity.com/market/priceoverview/?market_hash_name=AK-47%20%7C%20Redline%20%28Field-Tested%29&appid=730¤cy=1");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
std::cout << readBuffer << std::endl;
}
return 0;
}
But when I run it , nothing show up .
Any help is appreciated ,
Thanks
curl doesn't follow redirects by default, and the site you mention uses those.
I had to turn on CURLOPT_FOLLOWLOCATION to make it work:
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // redirects
// bonus:
curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, 1L); // corp. proxies etc.
Possible output:
{"success":true,"lowest_price":"$19.00","volume":"477","median_price":"$18.95"}
While debugging, you may want this option too to see what curl is up to:
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
I'm tring to create a telegram bot. I' m first trying to get updates from the bot like said in the Telegram API with /getUpdates method.
With Postman the request is working good and I have all the data in json format.
Using cUrl I have no response and res is 0. Here there is the snippet of code:
#include <iostream>
#include <curl/curl.h>
#include <string>
using namespace std;
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
void getUpdates()
{
std::string readBuffer;
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(curl, CURLOPT_URL, "http://api.telegram.org/BOTTOKEN/getUpdates");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_DEFAULT_PROTOCOL, "https");
struct curl_slist *headers = NULL;
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
std::cout << "Buffer content"<<readBuffer << std::endl;
long code = -1;
curl_easy_getinfo( curl, CURLINFO_RESPONSE_CODE, &code );
std::cout<<"HTTP Response Code: "<< code <<std::endl;
std::cout<<"Res: "<<res<<std::endl;
res = curl_easy_perform(curl);
}
}
int main()
{
getUpdates();
return 0;
}
Buffer content is empty and res is 0.
Could you give me any hints? Thank you!
I solved it! Here there is the updated code:
void getUpdates()
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(curl, CURLOPT_URL, "http://api.telegram.org/botyourtoken/getUpdates");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_DEFAULT_PROTOCOL, "https");
struct curl_slist *headers = NULL;
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
std::string readBuffer;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
std::cout << "Buffer content: "<<std::endl;
std::cout<<readBuffer << std::endl;
}
curl_easy_cleanup(curl);
}
Thank you!
I have POST request on python with a lot of settings, and I don't uderstand how their look like in curl.
data_str = '{' + '"username": "{}", "domain_id": {}, "password": {}'.format(login, domain_id, password) + '}'
try:
data = requests.post("https://example.com/v/session",
proxies=proxy,
verify=False,
data=data_str,
headers={"Content-Type": "application/json;charset=UTF-8",
"Accept": "application/json"})
if is_json(data.text):
print(data)
I find that url set parament CURLOPT_URL, headers - CURLOPT_HTTPHEADER. But how set proxy, verify, data ? How get json as in python ?
how to complete the code that it have the same result as in python:
CURL *curl = curl_easy_init();
struct curl_slist *list = NULL;
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://example.com");
list = curl_slist_append(list, "Shoesize: 10");
list = curl_slist_append(list, "Accept:");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
curl_easy_perform(curl);
curl_slist_free_all(list); /* free the list again */
}
In order to get the return data from the curl request, we need a callback function for the CURLOPT_WRITEFUNCTION option.
The proxy, data, verify parameters should be set as following :
#include <iostream>
#include <string>
#include <curl/curl.h>
size_t curlWriter(void *contents, size_t size, size_t nmemb, std::string *s)
{
size_t newLength = size*nmemb;
try
{
s->append((char*)contents, newLength);
}
catch(std::bad_alloc &e)
{
//memory problem
return 0;
}
return newLength;
}
int main()
{
CURL *curl;
CURLcode res;
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
if(curl)
{
std::string strResponse;
std::string strPostData = "my data post";
curl_easy_setopt(curl, CURLOPT_URL, "https://example.com/v/session");
curl_easy_setopt (curl, CURLOPT_VERBOSE, 1L);
//set the proxy
curl_easy_setopt(curl, CURLOPT_PROXY, "http://proxy.net");
curl_easy_setopt(curl, CURLOPT_PROXYPORT, 8080L);
//verify=False. SSL checking disabled
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
//set the callback function
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriter);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &strResponse);
/* size of the POST data */
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, strPostData.length() );
/* pass in a pointer to the data - libcurl will not copy */
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, strPostData.c_str() );
/* Execute the request */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
{
std::cerr << "CURL error : " << curl_easy_strerror(res) << std::endl;
}else {
std::cout << "CURL result : " << strResponse << std::endl;
}
curl_easy_cleanup(curl);
}
}
I am trying to download a .txt file from a server which I can access via the web browser on my raspberry pi.
Curl library gives segmentation error when I am trying to do this. Here is the code I am using.
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
int checkNewFiles(){
CURL *curl;
FILE *fp;
CURLcode res;
string url = "http://52.233.176.151:1880/files/device/software/text.txt";
char outfilename[FILENAME_MAX] = "/home/pi/Desktop/project/cpp/ab.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
I found the problem, what is url.c_str() doing?
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
change this to
curl_easy_setopt(curl, CURLOPT_URL, url);
Example : Curl program that download the text file.
Offcourse you need to add this neccessary header file here.
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main(void) {
CURL *curl;
FILE *fp;
CURLcode res;
const char *url = "http://localhost/yourfile.txt";
char outfilename[FILENAME_MAX] = "C:\\outfile.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); /* enable failure on http errors */
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
if(res != CURLE_OK) { /* check that the operation was successful */
printf("curl_easy_perform(): %s\n", curl_easy_strerror(res));
}
/* always cleanup */
curl_easy_cleanup(curl);
fclose(fp);
}
return 0;
}
I noticed you're not checking for errors after fopen. If it fails, it returns a NULL pointer, which would cause a segfault when curl attempts to write to it.
I'm not convinced that c_str() was the culprit to your segfault in the original question as I have used that in numerous applications with no problems.
I am trying to login to Twitter via CURL.
This is my code:
#include <iostream>
#include <string>
#include <curl\curl.h>
#include <sstream>
using std::string;
size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
std::string buf = std::string(static_cast<char *>(ptr), size * nmemb);
std::stringstream *response = static_cast<std::stringstream *>(stream);
response->write(buf.c_str(), (std::streamsize)buf.size());
return size * nmemb;
}
int main(int argc, char*argv[])
{
std::string target = "https://twitter.com/login";
CURL *curl;
CURLcode res;
std::stringstream response;
char* data = "session[username_or_email]=user&session[password]=pass";
struct curl_slist *chunk = NULL;
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, target.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, true);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
curl_global_cleanup();
}
std::cout << response.str() << std::endl;
cin.ignore();
return 0;
}
And I'm getting this message:
403 Forbidden: The server understood the request, but is refusing to
fulfill it
What should I do?
I suggest your request is (correctly) being identified as coming from a program not a human, and therefore denied.
It will be a matter of capturing all the right headers and cookies from a "real" (ie browser) session... But really, I think you'd be better off working with the Twitter api, as that is what it's designed for.