I have 2 files, grabURL.cpp and grabURL.hpp. When I compile the both with gcc it gives me multiple definitions errors.
grabURL.cpp
#include "grabURL.hpp"
size_t buffer(void *ptr, size_t size, size_t nmemb, void *stream) {
int numbytes = size*nmemb;
char lastchar = *((char *) ptr + numbytes - 1);
*((char *) ptr + numbytes - 1) = '\0';
ret.append((char *)ptr);
ret.append(1,lastchar);
*((char *) ptr + numbytes - 1) = lastchar;
return size*nmemb;
}
void grabURL(std::string url, std::string *theRet){
curl_easy_setopt(curl,CURLOPT_URL,url.c_str());
curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION, buffer);
res = curl_easy_perform(curl);
*theRet = ret;
}
grabURL.hpp:
#ifndef CURLURLGRAB_HPP
#define CURLURLGRAB_HPP
#include <string>
#include <curl/curl.h>
CURL * curl;
CURLcode res;
std::string ret;
size_t buffer(void *ptr, size_t size, size_t nmemb, void *stream);
void grabURL(std::string url,std::string* theRet);
#endif
I have no idea what I am doing wrong, I thought I did the ifndef thing correctly. What exactly is wrong?
This definitions in your header file:
CURL * curl;
CURLcode res;
std::string ret;
get defined when you include the file from the cpp file, and pressumably some other translation unit (perhaps main.cpp?)
They don't look like something that should be in a header file, but if they do then you can declare them extern within the header file and move the actual definitions to the cpp file.
Related
Developing from my previous question, wanted to generalize the behavior so that would download files within a specific data range.
So, modified the code to look like this
#include "pch.h"
#define _CRT_SECURE_NO_WARNINGS
#include <cstring>
#include <ctime>
#include <iostream>
#include <iostream>
#include <stdio.h>
#include <curl/curl.h>
#include <string.h>
#include <windows.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
void downloadFile(const char* url, const char* fname) {
CURL *curl;
FILE *fp;
CURLcode res;
curl = curl_easy_init();
if (curl) {
fp = fopen(fname, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
}
struct mytm : std::tm { // inherit "tm" to extend it a little
mytm(int year, int month, int day) : tm{}, current{} {
tm_year = year - 1900;
tm_mon = month - 1;
tm_mday = day;
tm_hour = 12;
current = mktime(this);
}
void add_days(int days) {
current += days * 60 * 60 * 24;
std::memcpy(this, std::localtime(¤t), sizeof(std::tm));
}
std::string strftime(const char* format) {
std::string retval(128, ' '); // a suitable sized buffer
std::size_t len = std::strftime(&retval[0], retval.size(), format, this);
retval.resize(len);
return retval;
}
std::time_t current;
};
int main(void) {
mytm date(2007, 1, 1);
for (int i = 0; i < 10; ++i) {
Sleep(1000);
std::string url =
date.strftime("http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=%d-%m-%Y&variation=PT");
std::string file_location =
date.strftime("C:/Users/molecoder/Desktop/file_%d_%m_%Y.xls");
// Pass std::string to a function that needs const char* - https://stackoverflow.com/a/347959/5675325
const char * url_final = url.c_str();
const char * file_location_final = file_location.c_str();
downloadFile(url_final, file_location_final);
}
}
While this downloads the a XLS file with the right name and data (so it's requesting correctly the URL)
it does only so for the initial date - the day, month and year are not changing.
You assign to date only once:
mytm date(2007, 1, 1);
And never modify that. So of course it doesn't change.
Update date before you format the new filename.
As sehe says, the problem is the date not being modified and so I needed to update date, like this
#include "pch.h"
#define _CRT_SECURE_NO_WARNINGS
#include <cstring>
#include <ctime>
#include <iostream>
#include <stdio.h>
#include <curl/curl.h>
#include <string>
#include <windows.h>
#include <cstdlib>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
void downloadFile(const char* url, const char* fname) {
CURL *curl;
FILE *fp;
CURLcode res;
curl = curl_easy_init();
if (curl) {
fp = fopen(fname, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
}
struct mytm : std::tm { // inherit "tm" to extend it a little
mytm(int year, int month, int day) : tm{}, current{} {
tm_year = year - 1900;
tm_mon = month - 1;
tm_mday = day;
tm_hour = 12;
current = mktime(this);
}
void add_days(int days) {
current += days * 60 * 60 * 24;
std::memcpy(this, std::localtime(¤t), sizeof(std::tm));
}
std::string strftime(const char* format) {
std::string retval(128, ' '); // a suitable sized buffer
std::size_t len = std::strftime(&retval[0], retval.size(), format, this);
retval.resize(len);
return retval;
}
std::time_t current;
};
int main(void) {
for (int year : {2018, 2019})
for (int month : {1, 11})
for (int day : {1, 31}) {
std::cout <<
mytm(year, month, day)
.strftime("http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=%d-%m-%Y&variation=PT")
<< std::endl;
Sleep(1000);
std::string url =
mytm(year, month, day).strftime("http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=%d-%m-%Y&variation=PT");
std::string file_location =
mytm(year, month, day).strftime("C:/Users/molecoder/Desktop/file_%d_%m_%Y.xls");
// Pass std::string to a function that needs const char* - https://stackoverflow.com/a/347959/5675325
const char * url_final = url.c_str();
const char * file_location_final = file_location.c_str();
downloadFile(url_final, file_location_final);
}
}
Notice that some of the files won't have data and would get the following result
Ocorreu um
erro.REN_CRG_Total::pr_REN_CRG_SelectTotalByDate_Excel::Error occured.
I'm trying to use libcurl (http://curl.haxx.se/libcurl/c/) for downloading data from a web, and store these data in a txt file , and here is my code:
// CLASS SinaStk
size_t save_data(char *buffer, size_t size, size_t nmemb, FILE* userdata){
locale loc = std::locale::global(std::locale("")); //TRY TO OPEN FILE WITH CHINESE
userdata = fopen(fpath.c_str(), "w");
if (userdata == NULL)
printf("File not open!\n");
locale::global(loc);
size_t writelen=size * nmemb;
fwrite(buffer, size, nmemb, userdata);
return writelen;
};
virtual void downloadUrl()
{
CURL* stkCURL=NULL;
CURLcode res;
FILE * fp=NULL;
curl_global_init(CURL_GLOBAL_WIN32);
stkCURL = curl_easy_init();
curl_easy_setopt(stkCURL, CURLOPT_URL,"http://hq.sinajs.cn/list=s_sh000001");
curl_easy_setopt(stkCURL, CURLOPT_WRITEFUNCTION, &SinaStk::save_data);
curl_easy_setopt(stkCURL, CURLOPT_WRITEDATA,fp);
res=curl_easy_perform(stkCURL); //<-STOP!!!!
fclose(fp);
curl_easy_cleanup(stkCURL);
curl_global_cleanup();
return;
};
and when I debug my code, it always stop and then jump to xstring:
size_type size() const _NOEXCEPT
{ // return length of sequence
return (this->_Mysize); // <-STOP!!!
}
0xC0000005: Access violation reading location 0x0000009E
I have no idea about the problem for almost a week. I am upset, I asked people around me and nobody knows why.
Thanks for reading, I am really confused.
=============
Problem is solved! Thanks you guys! now my code is:
//CLASS StkApiInfo
size_t writeData(char* buffer, size_t size, size_t nmemb){
if (stkFile.is_open()){
stkFile.close();
stkFile.clear();
};
fpath = "D:\\Code\\代码\\数据文件\\" + fname + ".txt";
stkFile.open(fpath.c_str(), ios::out);
//if (stkFile.is_open())
cout << buffer<<size<<nmemb;
stkFile << buffer<<endl;
stkFile.close();
stkFile.clear();
return size*nmemb;
};
//CLASS SinaStk : public StkApiInfo
static size_t save_data(char *buffer, size_t size, size_t nmemb, void* userdata){
SinaStk* self = (SinaStk*)userdata;
return self->writeData(buffer, size, nmemb);
};
virtual void downloadUrl()
{
CURL* stkCURL = NULL;
CURLcode res;
curl_global_init(CURL_GLOBAL_WIN32);
stkCURL = curl_easy_init();
if (stkCURL)
{
curl_easy_setopt(stkCURL, CURLOPT_URL, stkUrl.c_str());
curl_easy_setopt(stkCURL, CURLOPT_WRITEFUNCTION, &SinaStk::save_data);
curl_easy_setopt(stkCURL, CURLOPT_WRITEDATA, this);
res = curl_easy_perform(stkCURL);
//if (res != CURLE_OK)
curl_easy_cleanup(stkCURL);
curl_global_cleanup();
}
return;
};
Callback passed with CURLOPT_WRITEFUNCTION argument should be of type write_callback (with exact that signature) and therefore cannot be non-static class method. Usual workaround is to define callback as non-member or static method and pass this as an argument:
static size_t save_data(char *buffer, size_t size, size_t nmemb, void* userdata)
{
SinaStk* self = (SinaStk*) userdata;
return self->doStuff(buffer, size, nmemb);
}
virtual void downloadUrl()
{
//...
curl_easy_setopt(stkCURL, CURLOPT_WRITEFUNCTION, &SinaStk::save_data);
curl_easy_setopt(stkCURL, CURLOPT_WRITEDATA, this);
//...
}
If you need to access additional data (like FILE* in your example) you can either store it as class field or introduce temporary structure that would contain this and additional data fields and pass it's address as callback argument.
I am using http://ubuntuforums.org/showthread.php?t=781021 as a guide for how to write a c++ program that use libcurl to check a website and downloads the text to a c++ app. When I copy the code to my visual studio project, it works with only slight modification. However, I hit a snag trying to convert it to class based files.
I created a .h and .cpp files to run the query. What I didn't realize before was that handle_data was not a typical function, since it is not given any arguments or even () in the midway down Viewer.cpp. This is how the above site did it, and it worked, but I'm at a loss as to why or how to convert it now.
Does anyone know whats going on here or how to fix it?
Thanks!
Viewer.h:
#pragma once
#include <string>
#include "curl.h"
class Viewer
{
public:
Viewer(void);
~Viewer(void);
std::string view(std::string q);
private:
size_t handle_data(void *ptr, size_t size, size_t nmemb, void *stream);
std::string contents;
};
Viewer.cpp:
#include "stdafx.h"
#include "Viewer.h"
#include <iostream>
Viewer::Viewer(void)
{
std::cout << "ViewerCreated!\n";
}
Viewer::~Viewer(void)
{
}
size_t Viewer::handle_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
int numbytes = size*nmemb;
// The data is not null-terminated, so get the last character, and replace
// it with '\0'.
char lastchar = *((char *) ptr + numbytes - 1);
*((char *) ptr + numbytes - 1) = '\0';
contents.append((char *)ptr);
contents.append(1,lastchar);
*((char *) ptr + numbytes - 1) = lastchar; // Might not be necessary.
return size*nmemb;
}
std::string Viewer::view(std::string q)
{
char* url = "www.google.com";
CURL* curl = curl_easy_init();
if(curl)
{
// Tell libcurl the URL
curl_easy_setopt(curl,CURLOPT_URL, url);
// Tell libcurl what function to call when it has data
curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION,handle_data);
// Do it!
CURLcode res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
if (res == 0)
std::cout << contents << std::endl;
else
std::cerr << "Error: " << res << std::endl;
}
return contents;
}
You should not replace the last character of the chunk that you have received. Instead you should force that the buffer containing the appended blocks has a '\0' just after the end of the data. This way it can be printed, but if you are going to store that in a file, you should just write the bytes that you have received.
This has always worked for me:
struct MemoryStruct chunk;
chunk.memory=NULL;
chunk.size = 0;
...
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk);
And the callback:
struct MemoryStruct {
char *memory;
size_t size;
};
static void *myrealloc(void *ptr, size_t size)
{
if(ptr)
return realloc(ptr, size);
else
return malloc(size);
}
static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
{
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)data;
mem->memory = (char *)myrealloc(mem->memory, mem->size + realsize + 1);
if (mem->memory) {
memcpy(&(mem->memory[mem->size]), ptr, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
}
return realsize;
}
handle_data is used as a parameter for curl_easy_setopt. What that means is that you have to program what handle_data does when it receives the next data chunk (providing the data chunk is what curl will do for you).
handle_data is being passed to curl_easy_setopt as a callback function, which it will call when there's data available. Because of that, handle_data must be called as a static function (that is, one not receiving an object pointer, because cURL doesn't have one). However, you're not declaring it as static. Even if you do, you are accessing contents inside it, which is invalid.
i have python script that download exchange rates from web page, and i want make c++ program from that, here is what i have so far:
include iostream
include time.h
include stdio.h
include curl/curl.h
include curl/easy.h
include string
define CURL_STATICLIB
using namespace std;
void dat(string &d){
time_t rawtime;
struct tm * timeinfo;
char datum[80];
time ( &rawtime );
timeinfo=localtime(&rawtime);
strftime(datum,80,"%d%m%y",timeinfo);
d=datum;
}
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main()
{
string f;
dat(f);
string l1="http://www.hnb.hr/tecajn/f";
string l2=".dat";
string linkz=l1+f+l2;
cout << linkz;
CURL *curl;
FILE *fp;
CURLcode res;
char *url = linkz;
char outfilename[FILENAME_MAX] = "/home/tomi/data.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
return 0;
}
it give me this error when i try to compile, i found used algorythm for download txt, so i hope it is correct
If you'd have pointed out the line where you got the error, I wouldn't have had to track it down to:
string linkz=l1+f+l2;
...
char *url = linkz;
You can use c_str() to get a pointer to the const characters in the string. So this will do:
char const* url = linkz.c_str();
You could have that very same line in the setopt call, or have url be an std::string as well.
char *url = linkz; should be const char* url = linkz.c_str(); assuming you really need a C-style string for API reasons.
The problem is in this line:
char *url = linkz;
"links" is an std::string, but"url" is a char *. Try using the c_str method of string to get what you need like so:
const char * url = links.c_str();
I'm working on an application in C++ that threads and hands a bunch of threads URLs for cURL to download in parallel.
I'm employing a method that should be safe to download images and videos, etc. I uses memcpy instead of assuming the data is a string or character array.
I pass each thread a structure, thread_status, for a number of things. The structure lets the parent process know the thread is done downloading. It also stores the data cURL is downloading and keeps track of it's size as cURL returns more buffers for writing.
I pass a (void *) pointer that points to each structure that's allocated at initialization to each thread that does the downloading. The first page is downloaded properly, after that I keep getting errors from realloc().
Here is the simplest example that illustrates my problem. This sample is not multi-threaded but uses a similar structure to keep track of itself.
#include <string>
#include <assert.h>
#include <iostream>
#include <curl/curl.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#define NOT_READY 1
#define READY 0
using namespace std;
struct thread_status {
int id;
pthread_t *pid;
int readyState;
char *url;
void *data;
size_t bufferlen;
size_t writepos;
int initialized;
} ;
size_t static
writefunction( void *ptr, size_t size,
size_t nmemb, void *userdata)
{
size_t nbytes = size*nmemb;
struct thread_status **this_status;
this_status = (struct thread_status **) userdata;
if (!(*this_status)->initialized){
(*this_status)->data = (void *)malloc(1024);
(*this_status)->bufferlen = 1024;
(*this_status)->writepos = 0;
(*this_status)->initialized = true;
}
if ((*this_status)->bufferlen < ((*this_status)->writepos + nbytes)){
(*this_status)->bufferlen = (*this_status)->bufferlen + nbytes;
(*this_status)->data = realloc((*this_status)->data, (size_t) ((*this_status)->writepos + nbytes));
}
assert((*this_status)->data != NULL);
memcpy((*this_status)->data + (*this_status)->writepos, ptr, nbytes);
(*this_status)->writepos += nbytes;
return nbytes;
}
void *pull_data (void *my_struct){
struct thread_status *this_struct;
this_struct = (struct thread_status *) my_struct;
this_struct->initialized = false;
cout<<(char *)this_struct->url<<"\n";
CURL *curl;
curl = curl_easy_init();
size_t rc = 0;
while(true){
curl_easy_setopt(curl,
CURLOPT_WRITEFUNCTION, writefunction);
curl_easy_setopt(curl,
CURLOPT_WRITEDATA, (void *) &this_struct);
curl_easy_setopt(curl,
CURLOPT_NOSIGNAL, true);
curl_easy_setopt(curl,
CURLOPT_URL, (char *)this_struct->url);
if (curl_easy_perform(curl) != 0){
cout<<"curl did not perform\n";
exit(1);
} else {
if (this_struct->data != NULL){
// Use a binary write.
rc = fwrite(this_struct->data, this_struct->writepos, 1, stdout);
free(this_struct->data);
} else {
cout<<"Data is NULL\n";
}
}
// Tell the babysitter the thread is ready.
this_struct->readyState = READY;
// This would pause the thread until the parent thread has processed the data in it.
// while(this_struct->readyState == READY){;}
// Now get ready for another round!
this_struct->writepos = (size_t) 0;
this_struct->initialized = false;
this_struct->bufferlen = (size_t) 0;
break;
}
curl_easy_cleanup(curl);
return (void *)"a";
}
int main(){
char *urls[] = { "http://www.example.com/", "http://www.google.com", "http://www.touspassagers.com/", "http://www.facebook.com/" };
int i=0;
struct thread_status mystatuses[4];
for (i=0;i<4;i++){
struct thread_status my_status;
char *data;
my_status.id = i;
my_status.readyState = NOT_READY;
my_status.url = urls[i];
my_status.data = data;
my_status.bufferlen = 0;
my_status.writepos = 0;
my_status.initialized = false;
mystatuses[i] = my_status;
}
for (i=0;i<4;i++){
cout<<"pulling #"<<i<<"\n";
pull_data((void *)&mystatuses[i]);
}
}
If anyone can enlighten me as to the source of my error or a remedy for it I would appreciate it.
You might consider using valgrind to help locate the source of the memory problems.
Got it!
Apparently 1KB isn't enough memory to handle the first cURL buffer. I changed 1024 to nbytes and it works!
Before the memory memcpy put in the buffer ran over the allocated memory resulting in corruptions.
I did a post about it if anyone cares to see the full implementation:
http://www.touspassagers.com/2011/01/a-working-curlopt_writefunction-function-for-libcurl/