i'm a beginner in c++. I want to send request to a API, for this i use libcurl, and stock the response on a string and copy the string in a file. it is my test file :
#include <iostream>
#include <string>
#include <curl/curl.h>
#include <fstream>
int MyCurlObject::curlWriter(char *data, size_t size, size_t nmemb, std::string *buffer) {
int result = 0;
if (buffer != NULL) {
buffer->append(data, size * nmemb);
result = size * nmemb;
}
return result;
}
int main (){
std::string url = "https://www.google.com/";
std::string content;
curl = curl_easy_init();
if(!curl)
{
std::cerr << "impossible d'initialiser curl." << std::endl;
}
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &MyCurlObject::curlWriter);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
const CURLcode rc = curl_easy_perform(curl);
if( rc != CURLE_OK ) {
std::cout << "Error from cURL: " << curl_easy_strerror(rc) << std::endl;
}
std::ofstream file(fileName);
if(!file){
std::cerr << "can't open this file : " << fileName << std::endl;
}
file << content;
file.close();
return 0;
}
My files contain all of my string, but line in file not contain end of line symbol ( i display all of symbol with notepade++ and i just see CR in end of line ) and if i make this :
std::ifstream file(name);
if(file)
{
std::string crash;
int nbrOfLine = 0;
while(getline(file, crash))
{
std::cout << crash;
nbrOfLine++;
}
}
return 1 but my file contain 1500 lines.
thank you in advance
Related
I'm trying to build a small multithreading program which takes a subdomains and test them if they are alive on http or https, I've problem that's my program doesn't produce the correct output each time I get different output and also freeze and doesn't continue execution. I followed http://www.cplusplus.com/reference/thread/thread/thread/ when implementing the multithreading.
int main(int argc, char const *argv[] )
{
if (argc < 2){
cout << "Usage httplive <path to subdomains>" << endl;
}
ifstream http(argv[1]);
string line;
vector <std::thread> thread_pool;
while (getline(http, line)){
thread_pool.push_back(thread(httpTest,line, true));
thread_pool.push_back(thread(httpTest, line, false));
}
for (auto& t : thread_pool){
t.join();
}
return 0;
}
void httpTest(string line, bool Flag){
CURL *curl = curl_easy_init();
CURLcode res;
if (curl) {
line = Flag ? "https://" + line : "http://"+ line;
curl_easy_setopt(curl, CURLOPT_URL, const_cast<char*>(line.c_str()));
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
// curl_easy_setopt(curl, CURLOPT_TIMEOUT, 1L);
res = curl_easy_perform(curl);
// cout << res << endl;
if (res == CURLE_OK ) cout << line << endl;
}
curl_easy_cleanup(curl);
}
I downloaded the source code of Curl and built the library (libcurl.lib). Following is the code to read from the site and dump the contents.
The code works well for http sites and fails for https. I tried downloading openssl libraries but unable to link them as more linker errors are thrown.
What is the best solution to handle this?
#include "stdafx.h"
#include <iostream>
#include <string>
#include <curl/curl.h>
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
std::string readBuffer;
std::string curl_url = "https://www.example.com/";
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, curl_url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
std::cout << "Error from cURL: " << curl_easy_strerror(res) << std::endl;
}
curl_easy_cleanup(curl);
std::cout << "Finished reading from the website" << std::endl;
std::cout << readBuffer << std::endl;
}
return 0;
}
Essentially, I have a write callback that I set for CURLOPT_WRITEFUNCTION. I also set my pointer for writing the data to in CURLOPT_WRITEDATA.
I run an infinite while loop for around 5 seconds, then CURLOPT fails to right data to the void* up pointer defined in CURLOPT_WRITEDATA, then after that one failure it starts working again. I got tons of successful writes sprinkled with failures. Is this a memory issue, and if so, is there a way to circumvent reaching this issue altogether?
// these locations will likely be different on your local.
#include "../cygwin64/usr/include/curl/curl.h"
#include "../cygwin64/usr/include/json/json.h"
size_t write_callback(char *buf, size_t size, size_t nmemb, void* up) {
size_t num_bytes = size*nmemb;
std::string* data = (std::string*) up;
for(int i = 0; i < num_bytes; i++) {
data->push_back(buf[i]);
}
return num_bytes;
}
CURL* init_curl(struct curl_slist *headers, std::string* chunk) {
CURL *curl;
CURLcode res;
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) chunk);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
return curl;
}
bool curl_get(struct curl_slist *headers, const std::string& url,
Json::Value* json_res) {
// this is where my data should be written to
std::string data;
CURL* curl = init_curl(headers, &data);
CURLcode res;
bool success = true;
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
res = curl_easy_perform(curl);
if(res != CURLE_OK) {
std::cout << "Could not perform get for " << url << std::endl;
std::cout << curl_easy_strerror(res) << std::endl;
success = false;
} else {
Json::Value json_data;
Json::CharReaderBuilder json_reader;
std::istringstream stream_data(data);
std::string errs;
if(Json::parseFromStream(json_reader, stream_data, &json_data, &errs)) {
std::cout << "successfully parsed JSON data for: " << url << std::endl;
*json_res = json_data;
} else {
std::cout << "failed to parse JSON data for: " << url << std::endl;
std::cout << errs << std::endl;
std::cout << json_data << std::endl;
std::cout << "finished failing" << std::endl;
success = false;
}
}
} else {
success = false;
}
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
return success;
}
int main(int argc, char** argv) {
curl_global_init(CURL_GLOBAL_ALL);
struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "Accept: application/json");
while (true) {
Json::Value response;
if (curl_get(headers, "https://api.robinhood.com/quotes/?symbols=AMZN", &response)) {
std::cout << response << std::endl;
} else {
// reaches here sometimes because response (my data) is null
std::cout << "failed to get last trade price" << std::endl;
}
}
curl_global_cleanup();
return 0;
}
Expect no failures, but I get intermittent failures to right the data to my WRITEDATA chunk.
I have this code as below
#include <stdio.h>
#include <curl/curl.h>
#include <iostream>
using namespace std;
size_t read_file_data(char* buffer, size_t size,
size_t nitems, void* instream) {
printf("\nUpload file binary data");
cout << "Read" << nitems << " items by " << size << " bytes" << endl;
size_t retcode = fread(buffer, size, nitems,
static_cast<FILE*>(instream));
size_t retsize = size * retcode;
cout << "Read " << retsize << "bytes from file\n";
return retsize;
}
int main(void)
{
CURL* curl = curl_easy_init();
if (!curl) {
return 1;
}
curl_easy_setopt(curl, CURLOPT_URL, "http://172.30.191.145:3000/upload");
struct curl_slist* chunk = NULL;
chunk = curl_slist_append(chunk, "Transfer-Encoding: chunked");
chunk = curl_slist_append(chunk,
"Content-type:application/octet-stream");
chunk = curl_slist_append(chunk, "Session: test");
CURLcode result = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
FILE* fp =
fopen("/home/thinh/Documents/SDL/curlupload/testfile.txt","rb");
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_READFUNCTION,
&read_file_data);
curl_easy_setopt(curl, CURLOPT_READDATA, fp);
result = curl_easy_perform(curl);
cout << "curl result " << result << endl;
CURLcode get_info_result;
double val = 0;
get_info_result = curl_easy_getinfo(curl, CURLINFO_SIZE_UPLOAD, &val);
if (CURLE_OK == get_info_result) {
printf("Size of uploaded data: %0.0f bytes.\n", val);
} else {
printf("ERROR");
}
curl_easy_cleanup(curl);
curl_slist_free_all(chunk);
fclose(fp);
return 0;
}
After execute, the Size of uploaded data always larger than actual file size.
But the file after uploaded in server is correct(same with local file)
I use ll command to check file size
problem in this line
get_info_result = curl_easy_getinfo(curl, CURLINFO_SIZE_UPLOAD, &val);
For example: I upload local file text.txt 1022 bytes, but in log: "Size of uploaded data: 1034 bytes.", in server it is 1022 bytes.
text.txt content:
abcd123
abcd123
abcd123
I couldn't find any document talk detail about upload file size.
After change this line:
chunk = curl_slist_append(chunk, "Transfer-Encoding: chunked");
to
chunk = curl_slist_append(chunk, "Transfer-Encoding: ");
I get correct size in log.
The chunked encoding modifies the body of a message in order to transfer it as a series of chunks, each with its own size indicator, followed by an optional trailer containing entity-header fields Ref.
But I have to use chunked. Is there any way to get correct uploaded data
Hey guys I have the following issue. I've been using C++ to scrape website using to find 5 most frequent words in outputHTML which is string. Currently I have following code. Any hint would be awesome.
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://example.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &htmlOutput);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
std::cout << htmlOutput << std::endl;
}
Here's some hints for more awesomeness:
std::istringstream awsome_stream(web_text);
std::string word;
std::map<std::string, unsigned int> kewl_words;
while (awsome_stream >> word)
{
kewl_words[word]++;
}
std::cout << "Occurances of 'div': " << kewl_words["div"] << "\n";