How to send large string to the crow server with libcurl? - c++

This is my libcurl request (curl_example.cpp)
#include <iostream>
#include <string>
#include <curl/curl.h>
#include <vector>
#include <fstream> // std::ifstream
using namespace std;
using namespace cv;
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
std::string readBuffer;
int length=80000;
char c = 'z';
string encoded_png=std::string(length, c);
std::string url_data="http://0.0.0.0:18080/params?query="+encoded_png;
struct curl_slist *headerlist=NULL;
curl = curl_easy_init();
if(curl)
{
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
curl_easy_setopt(curl, CURLOPT_URL,url_data.c_str()) ;
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
std::cout <<"readBuffer:"<< readBuffer << std::endl;
}
return 0;
}
This is my Crow c++ server code (crow_example.cpp)
#include<iostream>
#include "crow.h"
#include<string>
using namespace std;
int main()
{
crow::SimpleApp app;
CROW_ROUTE(app, "/params")
([](const crow::request& req){
std::ostringstream os;
// To get a simple string from the url params
// To see it in action /params?foo='blabla'
os << "Params: " << req.url_params << "\n\n";
os << "The string received is " << (req.url_params.get("query") == nullptr ? "not " : "") << "found.\n";
return crow::response{os.str()};
});
app.port(18080).multithreaded().run();
}
with length=80000;
When I run the crow c++ server and query the server with libcurl I could able to receive the string in crow as well as get the response in curl.
with length = 85000;.//more than 80,000
I am not able to receive any string at the crow server as well there is no response from the libcurl request.
With the python flask server, I could receive the string with any length.
But the crow server is not working and I am not able to figure out what is the rootcause.
Any help or inputs appreciated.

Related

curl Programming to access https site

I downloaded the source code of Curl and built the library (libcurl.lib). Following is the code to read from the site and dump the contents.
The code works well for http sites and fails for https. I tried downloading openssl libraries but unable to link them as more linker errors are thrown.
What is the best solution to handle this?
#include "stdafx.h"
#include <iostream>
#include <string>
#include <curl/curl.h>
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
std::string readBuffer;
std::string curl_url = "https://www.example.com/";
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, curl_url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
std::cout << "Error from cURL: " << curl_easy_strerror(res) << std::endl;
}
curl_easy_cleanup(curl);
std::cout << "Finished reading from the website" << std::endl;
std::cout << readBuffer << std::endl;
}
return 0;
}

Writing Data Fails With CURLOPT_WRITEFUNCTION After Many Writeback Calls

Essentially, I have a write callback that I set for CURLOPT_WRITEFUNCTION. I also set my pointer for writing the data to in CURLOPT_WRITEDATA.
I run an infinite while loop for around 5 seconds, then CURLOPT fails to right data to the void* up pointer defined in CURLOPT_WRITEDATA, then after that one failure it starts working again. I got tons of successful writes sprinkled with failures. Is this a memory issue, and if so, is there a way to circumvent reaching this issue altogether?
// these locations will likely be different on your local.
#include "../cygwin64/usr/include/curl/curl.h"
#include "../cygwin64/usr/include/json/json.h"
size_t write_callback(char *buf, size_t size, size_t nmemb, void* up) {
size_t num_bytes = size*nmemb;
std::string* data = (std::string*) up;
for(int i = 0; i < num_bytes; i++) {
data->push_back(buf[i]);
}
return num_bytes;
}
CURL* init_curl(struct curl_slist *headers, std::string* chunk) {
CURL *curl;
CURLcode res;
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) chunk);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
return curl;
}
bool curl_get(struct curl_slist *headers, const std::string& url,
Json::Value* json_res) {
// this is where my data should be written to
std::string data;
CURL* curl = init_curl(headers, &data);
CURLcode res;
bool success = true;
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
res = curl_easy_perform(curl);
if(res != CURLE_OK) {
std::cout << "Could not perform get for " << url << std::endl;
std::cout << curl_easy_strerror(res) << std::endl;
success = false;
} else {
Json::Value json_data;
Json::CharReaderBuilder json_reader;
std::istringstream stream_data(data);
std::string errs;
if(Json::parseFromStream(json_reader, stream_data, &json_data, &errs)) {
std::cout << "successfully parsed JSON data for: " << url << std::endl;
*json_res = json_data;
} else {
std::cout << "failed to parse JSON data for: " << url << std::endl;
std::cout << errs << std::endl;
std::cout << json_data << std::endl;
std::cout << "finished failing" << std::endl;
success = false;
}
}
} else {
success = false;
}
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
return success;
}
int main(int argc, char** argv) {
curl_global_init(CURL_GLOBAL_ALL);
struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "Accept: application/json");
while (true) {
Json::Value response;
if (curl_get(headers, "https://api.robinhood.com/quotes/?symbols=AMZN", &response)) {
std::cout << response << std::endl;
} else {
// reaches here sometimes because response (my data) is null
std::cout << "failed to get last trade price" << std::endl;
}
}
curl_global_cleanup();
return 0;
}
Expect no failures, but I get intermittent failures to right the data to my WRITEDATA chunk.

c++ : libcurl request in file format error

i'm a beginner in c++. I want to send request to a API, for this i use libcurl, and stock the response on a string and copy the string in a file. it is my test file :
#include <iostream>
#include <string>
#include <curl/curl.h>
#include <fstream>
int MyCurlObject::curlWriter(char *data, size_t size, size_t nmemb, std::string *buffer) {
int result = 0;
if (buffer != NULL) {
buffer->append(data, size * nmemb);
result = size * nmemb;
}
return result;
}
int main (){
std::string url = "https://www.google.com/";
std::string content;
curl = curl_easy_init();
if(!curl)
{
std::cerr << "impossible d'initialiser curl." << std::endl;
}
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &MyCurlObject::curlWriter);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
const CURLcode rc = curl_easy_perform(curl);
if( rc != CURLE_OK ) {
std::cout << "Error from cURL: " << curl_easy_strerror(rc) << std::endl;
}
std::ofstream file(fileName);
if(!file){
std::cerr << "can't open this file : " << fileName << std::endl;
}
file << content;
file.close();
return 0;
}
My files contain all of my string, but line in file not contain end of line symbol ( i display all of symbol with notepade++ and i just see CR in end of line ) and if i make this :
std::ifstream file(name);
if(file)
{
std::string crash;
int nbrOfLine = 0;
while(getline(file, crash))
{
std::cout << crash;
nbrOfLine++;
}
}
return 1 but my file contain 1500 lines.
thank you in advance

Downloading an image from twitter with libcurl C++

I'm making a program that takes a person's tweet, and if it contains an image, download it.
Why can I download an image from this URL (Example #1)
http://www.google.co.id/intl/en_com/images/logo_plain.png
And not from this URL (Example #2)
https://www.google.com/imgres?imgurl=https://pbs.twimg.com/media/DR-kkH4XcAAQ-vc.jpg&imgrefurl=https://twitter.com/officialmcafee/status/945655402276024320&h=1200&w=992&tbnid=0q3B6ZB_UxjRIM&tbnh=247&tbnw=204&usg=__xvjbjSSMvuImESBLVvBBrUagUe8=&docid=vdqkoUmaefYoFM
Example #1
#include <iostream>
#include <curl/curl.h>
using namespace std;
int main()
{
CURL *image;
CURLcode imgresult;
FILE *fp = nullptr;
const char *url = "http://www.google.co.id/intl/en_com/images/logo_plain.png";
image = curl_easy_init();
if (image)
{
// Open file
fp = fopen("img.png", "wb");
if (fp == NULL) cout << "File cannot be opened";
curl_easy_setopt(image, CURLOPT_WRITEFUNCTION, NULL);
curl_easy_setopt(image, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(image, CURLOPT_URL, url);
// Grab image
imgresult = curl_easy_perform(image);
if (imgresult)
cout << "Cannot grab the image!\n";
}
// Clean up the resources
curl_easy_cleanup(image);
// Close the file
fclose(fp);
system("pause");
return 0;
}
Example #2
#include <iostream>
#include <curl/curl.h>
using namespace std;
int main()
{
CURL *image;
CURLcode imgresult;
FILE *fp = nullptr;
const char *url = "https://www.google.com/imgres?imgurl=https://pbs.twimg.com/media/DR-kkH4XcAAQ-vc.jpg&imgrefurl=https://twitter.com/officialmcafee/status/945655402276024320&h=1200&w=992&tbnid=0q3B6ZB_UxjRIM&tbnh=247&tbnw=204&usg=__xvjbjSSMvuImESBLVvBBrUagUe8=&docid=vdqkoUmaefYoFM";
image = curl_easy_init();
if (image)
{
// Open file
fp = fopen("img.png", "wb");
if (fp == NULL) cout << "File cannot be opened";
curl_easy_setopt(image, CURLOPT_WRITEFUNCTION, NULL);
curl_easy_setopt(image, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(image, CURLOPT_URL, url);
// Grab image
imgresult = curl_easy_perform(image);
if (imgresult)
cout << "Cannot grab the image!\n";
}
// Clean up the resources
curl_easy_cleanup(image);
// Close the file
fclose(fp);
system("pause");
return 0;
}
First, this is not a link to an image. This is html page. Note, that your code doesn't download any image the html page refers to, but just html page.
Second, you are not following redirects. Add one more option:
curl_easy_setopt(image, CURLOPT_FOLLOWLOCATION, 1);
Third, you'd better to pretend to be a browser:
curl_easy_setopt(image, CURLOPT_USERAGENT, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36");
Once I added both options, I managed to download your link.

for loop Download file from string link

I need to batch download a file from a link and this link is in a string, how do I do it? I downloaded curl but I don't know how to use it.
string goes this way:
www.example.com/item1.jpeg
www.example.com/item2.jpeg
and so on.
I don't need to change the output names, they can stay as they are.
I'm using this:
CURL curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, c_str(link));
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
}
But I'm getting the error:
[Error] 'c_str' was not declared in this scope
My whole script is:
#include <iostream>
#include <string>
#include <stdio.h>
#include <stdlib.h>
#include <curl/curl.h>
using namespace std;
int main ()
{
char buffer[21];
int start;
int end;
int counter;
string site;
site = "http://www.example.com/";
string extension;
extension= ".jpeg";
string link;
cout << "Start: ";
cin >> start;
cout << "End: ";
cin >> end;
for (counter=start; counter<=end; counter++)
{
std::string link = site+itoa(counter, buffer, 10)+extension;
cout << link;
cout << "\n";
//////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////
CURL curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, link.c_str());
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
}
//////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////
}
return 0;
}
The error is still there.
The error regarding c_str has nothing to do with curl. Instead, it indicates you have not used C++ strings correctly. Reviewing the documents one can see that c_str is a method of the string object.
http://www.cplusplus.com/reference/string/string/c_str/
So, in all likelihood, you need to have something of the following form:
#include <string>
#include <curl/curl.h>
int main () {
std::string link ("http://www.example.com/foo1.jpg");
CURL curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, link.c_str());
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
}
}