I'm making a program that takes a person's tweet, and if it contains an image, download it.
Why can I download an image from this URL (Example #1)
http://www.google.co.id/intl/en_com/images/logo_plain.png
And not from this URL (Example #2)
https://www.google.com/imgres?imgurl=https://pbs.twimg.com/media/DR-kkH4XcAAQ-vc.jpg&imgrefurl=https://twitter.com/officialmcafee/status/945655402276024320&h=1200&w=992&tbnid=0q3B6ZB_UxjRIM&tbnh=247&tbnw=204&usg=__xvjbjSSMvuImESBLVvBBrUagUe8=&docid=vdqkoUmaefYoFM
Example #1
#include <iostream>
#include <curl/curl.h>
using namespace std;
int main()
{
CURL *image;
CURLcode imgresult;
FILE *fp = nullptr;
const char *url = "http://www.google.co.id/intl/en_com/images/logo_plain.png";
image = curl_easy_init();
if (image)
{
// Open file
fp = fopen("img.png", "wb");
if (fp == NULL) cout << "File cannot be opened";
curl_easy_setopt(image, CURLOPT_WRITEFUNCTION, NULL);
curl_easy_setopt(image, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(image, CURLOPT_URL, url);
// Grab image
imgresult = curl_easy_perform(image);
if (imgresult)
cout << "Cannot grab the image!\n";
}
// Clean up the resources
curl_easy_cleanup(image);
// Close the file
fclose(fp);
system("pause");
return 0;
}
Example #2
#include <iostream>
#include <curl/curl.h>
using namespace std;
int main()
{
CURL *image;
CURLcode imgresult;
FILE *fp = nullptr;
const char *url = "https://www.google.com/imgres?imgurl=https://pbs.twimg.com/media/DR-kkH4XcAAQ-vc.jpg&imgrefurl=https://twitter.com/officialmcafee/status/945655402276024320&h=1200&w=992&tbnid=0q3B6ZB_UxjRIM&tbnh=247&tbnw=204&usg=__xvjbjSSMvuImESBLVvBBrUagUe8=&docid=vdqkoUmaefYoFM";
image = curl_easy_init();
if (image)
{
// Open file
fp = fopen("img.png", "wb");
if (fp == NULL) cout << "File cannot be opened";
curl_easy_setopt(image, CURLOPT_WRITEFUNCTION, NULL);
curl_easy_setopt(image, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(image, CURLOPT_URL, url);
// Grab image
imgresult = curl_easy_perform(image);
if (imgresult)
cout << "Cannot grab the image!\n";
}
// Clean up the resources
curl_easy_cleanup(image);
// Close the file
fclose(fp);
system("pause");
return 0;
}
First, this is not a link to an image. This is html page. Note, that your code doesn't download any image the html page refers to, but just html page.
Second, you are not following redirects. Add one more option:
curl_easy_setopt(image, CURLOPT_FOLLOWLOCATION, 1);
Third, you'd better to pretend to be a browser:
curl_easy_setopt(image, CURLOPT_USERAGENT, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36");
Once I added both options, I managed to download your link.
Related
I'm trying to build a small multithreading program which takes a subdomains and test them if they are alive on http or https, I've problem that's my program doesn't produce the correct output each time I get different output and also freeze and doesn't continue execution. I followed http://www.cplusplus.com/reference/thread/thread/thread/ when implementing the multithreading.
int main(int argc, char const *argv[] )
{
if (argc < 2){
cout << "Usage httplive <path to subdomains>" << endl;
}
ifstream http(argv[1]);
string line;
vector <std::thread> thread_pool;
while (getline(http, line)){
thread_pool.push_back(thread(httpTest,line, true));
thread_pool.push_back(thread(httpTest, line, false));
}
for (auto& t : thread_pool){
t.join();
}
return 0;
}
void httpTest(string line, bool Flag){
CURL *curl = curl_easy_init();
CURLcode res;
if (curl) {
line = Flag ? "https://" + line : "http://"+ line;
curl_easy_setopt(curl, CURLOPT_URL, const_cast<char*>(line.c_str()));
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
// curl_easy_setopt(curl, CURLOPT_TIMEOUT, 1L);
res = curl_easy_perform(curl);
// cout << res << endl;
if (res == CURLE_OK ) cout << line << endl;
}
curl_easy_cleanup(curl);
}
I downloaded the source code of Curl and built the library (libcurl.lib). Following is the code to read from the site and dump the contents.
The code works well for http sites and fails for https. I tried downloading openssl libraries but unable to link them as more linker errors are thrown.
What is the best solution to handle this?
#include "stdafx.h"
#include <iostream>
#include <string>
#include <curl/curl.h>
static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
std::string readBuffer;
std::string curl_url = "https://www.example.com/";
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, curl_url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
std::cout << "Error from cURL: " << curl_easy_strerror(res) << std::endl;
}
curl_easy_cleanup(curl);
std::cout << "Finished reading from the website" << std::endl;
std::cout << readBuffer << std::endl;
}
return 0;
}
i'm a beginner in c++. I want to send request to a API, for this i use libcurl, and stock the response on a string and copy the string in a file. it is my test file :
#include <iostream>
#include <string>
#include <curl/curl.h>
#include <fstream>
int MyCurlObject::curlWriter(char *data, size_t size, size_t nmemb, std::string *buffer) {
int result = 0;
if (buffer != NULL) {
buffer->append(data, size * nmemb);
result = size * nmemb;
}
return result;
}
int main (){
std::string url = "https://www.google.com/";
std::string content;
curl = curl_easy_init();
if(!curl)
{
std::cerr << "impossible d'initialiser curl." << std::endl;
}
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &MyCurlObject::curlWriter);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
const CURLcode rc = curl_easy_perform(curl);
if( rc != CURLE_OK ) {
std::cout << "Error from cURL: " << curl_easy_strerror(rc) << std::endl;
}
std::ofstream file(fileName);
if(!file){
std::cerr << "can't open this file : " << fileName << std::endl;
}
file << content;
file.close();
return 0;
}
My files contain all of my string, but line in file not contain end of line symbol ( i display all of symbol with notepade++ and i just see CR in end of line ) and if i make this :
std::ifstream file(name);
if(file)
{
std::string crash;
int nbrOfLine = 0;
while(getline(file, crash))
{
std::cout << crash;
nbrOfLine++;
}
}
return 1 but my file contain 1500 lines.
thank you in advance
I have this code as below
#include <stdio.h>
#include <curl/curl.h>
#include <iostream>
using namespace std;
size_t read_file_data(char* buffer, size_t size,
size_t nitems, void* instream) {
printf("\nUpload file binary data");
cout << "Read" << nitems << " items by " << size << " bytes" << endl;
size_t retcode = fread(buffer, size, nitems,
static_cast<FILE*>(instream));
size_t retsize = size * retcode;
cout << "Read " << retsize << "bytes from file\n";
return retsize;
}
int main(void)
{
CURL* curl = curl_easy_init();
if (!curl) {
return 1;
}
curl_easy_setopt(curl, CURLOPT_URL, "http://172.30.191.145:3000/upload");
struct curl_slist* chunk = NULL;
chunk = curl_slist_append(chunk, "Transfer-Encoding: chunked");
chunk = curl_slist_append(chunk,
"Content-type:application/octet-stream");
chunk = curl_slist_append(chunk, "Session: test");
CURLcode result = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
FILE* fp =
fopen("/home/thinh/Documents/SDL/curlupload/testfile.txt","rb");
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_READFUNCTION,
&read_file_data);
curl_easy_setopt(curl, CURLOPT_READDATA, fp);
result = curl_easy_perform(curl);
cout << "curl result " << result << endl;
CURLcode get_info_result;
double val = 0;
get_info_result = curl_easy_getinfo(curl, CURLINFO_SIZE_UPLOAD, &val);
if (CURLE_OK == get_info_result) {
printf("Size of uploaded data: %0.0f bytes.\n", val);
} else {
printf("ERROR");
}
curl_easy_cleanup(curl);
curl_slist_free_all(chunk);
fclose(fp);
return 0;
}
After execute, the Size of uploaded data always larger than actual file size.
But the file after uploaded in server is correct(same with local file)
I use ll command to check file size
problem in this line
get_info_result = curl_easy_getinfo(curl, CURLINFO_SIZE_UPLOAD, &val);
For example: I upload local file text.txt 1022 bytes, but in log: "Size of uploaded data: 1034 bytes.", in server it is 1022 bytes.
text.txt content:
abcd123
abcd123
abcd123
I couldn't find any document talk detail about upload file size.
After change this line:
chunk = curl_slist_append(chunk, "Transfer-Encoding: chunked");
to
chunk = curl_slist_append(chunk, "Transfer-Encoding: ");
I get correct size in log.
The chunked encoding modifies the body of a message in order to transfer it as a series of chunks, each with its own size indicator, followed by an optional trailer containing entity-header fields Ref.
But I have to use chunked. Is there any way to get correct uploaded data
I need to batch download a file from a link and this link is in a string, how do I do it? I downloaded curl but I don't know how to use it.
string goes this way:
www.example.com/item1.jpeg
www.example.com/item2.jpeg
and so on.
I don't need to change the output names, they can stay as they are.
I'm using this:
CURL curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, c_str(link));
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
}
But I'm getting the error:
[Error] 'c_str' was not declared in this scope
My whole script is:
#include <iostream>
#include <string>
#include <stdio.h>
#include <stdlib.h>
#include <curl/curl.h>
using namespace std;
int main ()
{
char buffer[21];
int start;
int end;
int counter;
string site;
site = "http://www.example.com/";
string extension;
extension= ".jpeg";
string link;
cout << "Start: ";
cin >> start;
cout << "End: ";
cin >> end;
for (counter=start; counter<=end; counter++)
{
std::string link = site+itoa(counter, buffer, 10)+extension;
cout << link;
cout << "\n";
//////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////
CURL curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, link.c_str());
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
}
//////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////
}
return 0;
}
The error is still there.
The error regarding c_str has nothing to do with curl. Instead, it indicates you have not used C++ strings correctly. Reviewing the documents one can see that c_str is a method of the string object.
http://www.cplusplus.com/reference/string/string/c_str/
So, in all likelihood, you need to have something of the following form:
#include <string>
#include <curl/curl.h>
int main () {
std::string link ("http://www.example.com/foo1.jpg");
CURL curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, link.c_str());
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
}
}