I am trying to downloading a lot of files from a website. I am recursively going from page to page and extract link from there. And after I downloaded
543 files (I've tried several times and the number is always the same),
I got a error code 2, which refers to CURLE_FAILED_INIT.
Piece of code, which downloaded a file (copy-paste from one of the tutorial):
CURLcode curl_read(const std::string& url, std::string& buffer, long timeout = 30) {
CURLcode code(CURLE_FAILED_INIT);
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
if(curl) {
if(CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_URL, url.c_str()))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, string_write))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout))) {
code = curl_easy_perform(curl);
}
else {
std::cout << "Error inside setting curl" << std::endl;
}
curl_easy_cleanup(curl);
}
else {
std::cout << "Error inside curl_easy_init" << std::endl;
}
return code;
}
Before this code run, I called
curl_global_init(CURL_GLOBAL_ALL);
Related
In a C ++ program I need to send an http message using Curl to another machine where is running a REST server written in C ++, and then collect the response from that server.
I am new to using Curl and have been looking at documentation and various examples on the web.
The source code of my program is the one attached below.
Sending the message to the server works, but I do not get the response the server sends me (not a timeout problem).
I would appreciate help with this, as I am not sure the code I use to get the response from the server is correct.
#include <stdio.h>
#include <iostream>
#include <unistd.h>
#include <sstream>
#include <curl/curl.h>
size_t getAnswerFunction(void* ptr, size_t size, size_t nmemb, std::string* data) {
data->append((char*)ptr, size * nmemb);
return size * nmemb;
}
void sendDataAndGetAnswer(std::string data)
{
CURL *curl;
CURLcode res;
struct curl_slist *httpHeaders=NULL;
curl = curl_easy_init();
if (curl)
{
curl_global_init(CURL_GLOBAL_ALL);
curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.0.100:15000");
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str());
httpHeaders = curl_slist_append(httpHeaders, "MyProgram-Header");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, httpHeaders);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "MyAgent/1.0");
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 1000);
// .........................................
// MyProgram sends data.
// This works right.
// .........................................
res = curl_easy_perform(curl);
if (res != CURLE_OK)
std::cout << "curl_easy_perform() failed to send message: " << curl_easy_strerror(res) << std::endl;
// .........................................
// MyProgram get answer.
// This does not work.
// .........................................
std::string response_string;
curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.0.100:15000");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, getAnswerFunction);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string);
// getAnswerFunction waits at most 5000 ms.
// Afterwards, the flow of the program should continue even if there is no response.
curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 5000);
res = curl_easy_perform(curl);
if (res == CURLE_OK)
std::cout << response_string;
else
std::cout << "curl_easy_perform() failed to get answer: " << curl_easy_strerror(res) << std::endl;
curl_easy_cleanup(curl);
curl_slist_free_all(httpHeaders);
}
}
int main(void)
{
while (1)
{
int valueVar = 0;
// Execute various statements and perform calculations.
// ...
// Calculate value here.
// ...
std::string msgToSend("CONTITION-IS-TRUE");
if (valueVar = 5)
sendDataAndGetAnswer(msgToSend);
sleep(10);
}
return 0;
}
I'm tring to create a telegram bot. I' m first trying to get updates from the bot like said in the Telegram API with /getUpdates method.
With Postman the request is working good and I have all the data in json format.
Using cUrl I have no response and res is 0. Here there is the snippet of code:
#include <iostream>
#include <curl/curl.h>
#include <string>
using namespace std;
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
void getUpdates()
{
std::string readBuffer;
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(curl, CURLOPT_URL, "http://api.telegram.org/BOTTOKEN/getUpdates");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_DEFAULT_PROTOCOL, "https");
struct curl_slist *headers = NULL;
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
std::cout << "Buffer content"<<readBuffer << std::endl;
long code = -1;
curl_easy_getinfo( curl, CURLINFO_RESPONSE_CODE, &code );
std::cout<<"HTTP Response Code: "<< code <<std::endl;
std::cout<<"Res: "<<res<<std::endl;
res = curl_easy_perform(curl);
}
}
int main()
{
getUpdates();
return 0;
}
Buffer content is empty and res is 0.
Could you give me any hints? Thank you!
I solved it! Here there is the updated code:
void getUpdates()
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(curl, CURLOPT_URL, "http://api.telegram.org/botyourtoken/getUpdates");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_DEFAULT_PROTOCOL, "https");
struct curl_slist *headers = NULL;
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
std::string readBuffer;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
std::cout << "Buffer content: "<<std::endl;
std::cout<<readBuffer << std::endl;
}
curl_easy_cleanup(curl);
}
Thank you!
I have POST request on python with a lot of settings, and I don't uderstand how their look like in curl.
data_str = '{' + '"username": "{}", "domain_id": {}, "password": {}'.format(login, domain_id, password) + '}'
try:
data = requests.post("https://example.com/v/session",
proxies=proxy,
verify=False,
data=data_str,
headers={"Content-Type": "application/json;charset=UTF-8",
"Accept": "application/json"})
if is_json(data.text):
print(data)
I find that url set parament CURLOPT_URL, headers - CURLOPT_HTTPHEADER. But how set proxy, verify, data ? How get json as in python ?
how to complete the code that it have the same result as in python:
CURL *curl = curl_easy_init();
struct curl_slist *list = NULL;
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://example.com");
list = curl_slist_append(list, "Shoesize: 10");
list = curl_slist_append(list, "Accept:");
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
curl_easy_perform(curl);
curl_slist_free_all(list); /* free the list again */
}
In order to get the return data from the curl request, we need a callback function for the CURLOPT_WRITEFUNCTION option.
The proxy, data, verify parameters should be set as following :
#include <iostream>
#include <string>
#include <curl/curl.h>
size_t curlWriter(void *contents, size_t size, size_t nmemb, std::string *s)
{
size_t newLength = size*nmemb;
try
{
s->append((char*)contents, newLength);
}
catch(std::bad_alloc &e)
{
//memory problem
return 0;
}
return newLength;
}
int main()
{
CURL *curl;
CURLcode res;
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
if(curl)
{
std::string strResponse;
std::string strPostData = "my data post";
curl_easy_setopt(curl, CURLOPT_URL, "https://example.com/v/session");
curl_easy_setopt (curl, CURLOPT_VERBOSE, 1L);
//set the proxy
curl_easy_setopt(curl, CURLOPT_PROXY, "http://proxy.net");
curl_easy_setopt(curl, CURLOPT_PROXYPORT, 8080L);
//verify=False. SSL checking disabled
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
//set the callback function
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriter);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &strResponse);
/* size of the POST data */
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, strPostData.length() );
/* pass in a pointer to the data - libcurl will not copy */
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, strPostData.c_str() );
/* Execute the request */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
{
std::cerr << "CURL error : " << curl_easy_strerror(res) << std::endl;
}else {
std::cout << "CURL result : " << strResponse << std::endl;
}
curl_easy_cleanup(curl);
}
}
Sure I am missing something obvious, but trying to make a curl get request and return the response to a variable.
size_t write_to_string(void *ptr, size_t size, size_t nmemb, std::string stream){
size_t realsize = size * nmemb;
std::string temp(static_cast<const char*>(ptr), realsize);
cout << temp << endl;
return realsize;
}
CURLcode curl_get(const std::string& url, std::ostream& os, long timeout = 30){
CURLcode code(CURLE_FAILED_INIT);
CURL* curl = curl_easy_init();
string *response;
if (curl){
if (CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_to_string))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_FILE, &os))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_WRITEDATA, response))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_URL, url.c_str())))
{
code = curl_easy_perform(curl);
}
curl_easy_cleanup(curl);
}
std::string a = *response;
return code;
}
response is always either null or an out of bounds pointer from the dozen ways I have tried. The cout does show the correct response.
All I am trying to do is create a generic function for all curl get calls and have it return the get data. New to c, so sorry if I am overlooking something simple.
You declare write_to_string to accept std::string but you pass it a pointer to std::string (via CURLOPT_WRITEDATA).
You don't do anything with std::string stream in write_to_string.
You declare response as a std::string* but do not initialize it, so you have dangling pointer.
I'm trying to fetch website source using C++ and libcurl in Qt 4.8. I'm new to Qt and C++.
However, Im getting an error message
Error Message -
Starting C:*****.exe...
The program has unexpectedly finished.
C:*****.exe exited with code -1073741819
#include <curl/include/curl/curl.h>
#include <fstream>
#include <sstream>
#include <iostream>
static size_t data_write(void* buf, size_t size, size_t nmemb, void* userp)
{
if(userp)
{
std::ostream& os = *static_cast<std::ostream*>(userp);
std::streamsize len = size * nmemb;
if(os.write(static_cast<char*>(buf), len))
return len;
}
return 0;
}
CURLcode curl_read(const std::string& url, std::ostream& os, long timeout = 30)
{
CURLcode code(CURLE_FAILED_INIT);
CURL* curl = curl_easy_init();
if(curl)
{
if(CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &data_write))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_FILE, &os))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout))
&& CURLE_OK == (code = curl_easy_setopt(curl, CURLOPT_URL, url.c_str())))
{
code = curl_easy_perform(curl);
}
curl_easy_cleanup(curl);
}
return code;
}
int main()
{
curl_global_init(CURL_GLOBAL_ALL);
if(CURLE_OK == curl_read("http://google.com", std::cout))
{
// Web page successfully written to standard output (console?)
}
curl_global_cleanup();
return 0;
}
Whats the problem with the code above ?
P.S - Above code is not mine.
Don't bother using external libraries use what Qt has got. And it has plenty !
Here is a simple exemple on how you can get a webpage source!
void MainWindow::on_pushButton_clicked()
{
QNetworkAccessManager * mgr = new QNetworkAccessManager(this);
connect(mgr,SIGNAL(finished(QNetworkReply*)),this,SLOT(onfinish(QNetworkReply*)));
connect(mgr,SIGNAL(finished(QNetworkReply*)),mgr,SLOT(deleteLater()));
mgr->get(QNetworkRequest(QUrl("http://www.google.com")));
}
void MainWindow::onfinish(QNetworkReply *rep)
{
QByteArray bts = rep->readAll();
QString str(bts);
QMessageBox::information(this,"sal",str,"ok");
}
And if you want to create a blocking function check out this tutorial over here.
Hope it helps!