i have python script that download exchange rates from web page, and i want make c++ program from that, here is what i have so far:
include iostream
include time.h
include stdio.h
include curl/curl.h
include curl/easy.h
include string
define CURL_STATICLIB
using namespace std;
void dat(string &d){
time_t rawtime;
struct tm * timeinfo;
char datum[80];
time ( &rawtime );
timeinfo=localtime(&rawtime);
strftime(datum,80,"%d%m%y",timeinfo);
d=datum;
}
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main()
{
string f;
dat(f);
string l1="http://www.hnb.hr/tecajn/f";
string l2=".dat";
string linkz=l1+f+l2;
cout << linkz;
CURL *curl;
FILE *fp;
CURLcode res;
char *url = linkz;
char outfilename[FILENAME_MAX] = "/home/tomi/data.txt";
curl = curl_easy_init();
if (curl) {
fp = fopen(outfilename,"wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
return 0;
}
it give me this error when i try to compile, i found used algorythm for download txt, so i hope it is correct
If you'd have pointed out the line where you got the error, I wouldn't have had to track it down to:
string linkz=l1+f+l2;
...
char *url = linkz;
You can use c_str() to get a pointer to the const characters in the string. So this will do:
char const* url = linkz.c_str();
You could have that very same line in the setopt call, or have url be an std::string as well.
char *url = linkz; should be const char* url = linkz.c_str(); assuming you really need a C-style string for API reasons.
The problem is in this line:
char *url = linkz;
"links" is an std::string, but"url" is a char *. Try using the c_str method of string to get what you need like so:
const char * url = links.c_str();
Related
Developing from my previous question, wanted to generalize the behavior so that would download files within a specific data range.
So, modified the code to look like this
#include "pch.h"
#define _CRT_SECURE_NO_WARNINGS
#include <cstring>
#include <ctime>
#include <iostream>
#include <iostream>
#include <stdio.h>
#include <curl/curl.h>
#include <string.h>
#include <windows.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
void downloadFile(const char* url, const char* fname) {
CURL *curl;
FILE *fp;
CURLcode res;
curl = curl_easy_init();
if (curl) {
fp = fopen(fname, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
}
struct mytm : std::tm { // inherit "tm" to extend it a little
mytm(int year, int month, int day) : tm{}, current{} {
tm_year = year - 1900;
tm_mon = month - 1;
tm_mday = day;
tm_hour = 12;
current = mktime(this);
}
void add_days(int days) {
current += days * 60 * 60 * 24;
std::memcpy(this, std::localtime(¤t), sizeof(std::tm));
}
std::string strftime(const char* format) {
std::string retval(128, ' '); // a suitable sized buffer
std::size_t len = std::strftime(&retval[0], retval.size(), format, this);
retval.resize(len);
return retval;
}
std::time_t current;
};
int main(void) {
mytm date(2007, 1, 1);
for (int i = 0; i < 10; ++i) {
Sleep(1000);
std::string url =
date.strftime("http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=%d-%m-%Y&variation=PT");
std::string file_location =
date.strftime("C:/Users/molecoder/Desktop/file_%d_%m_%Y.xls");
// Pass std::string to a function that needs const char* - https://stackoverflow.com/a/347959/5675325
const char * url_final = url.c_str();
const char * file_location_final = file_location.c_str();
downloadFile(url_final, file_location_final);
}
}
While this downloads the a XLS file with the right name and data (so it's requesting correctly the URL)
it does only so for the initial date - the day, month and year are not changing.
You assign to date only once:
mytm date(2007, 1, 1);
And never modify that. So of course it doesn't change.
Update date before you format the new filename.
As sehe says, the problem is the date not being modified and so I needed to update date, like this
#include "pch.h"
#define _CRT_SECURE_NO_WARNINGS
#include <cstring>
#include <ctime>
#include <iostream>
#include <stdio.h>
#include <curl/curl.h>
#include <string>
#include <windows.h>
#include <cstdlib>
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
void downloadFile(const char* url, const char* fname) {
CURL *curl;
FILE *fp;
CURLcode res;
curl = curl_easy_init();
if (curl) {
fp = fopen(fname, "wb");
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
fclose(fp);
}
}
struct mytm : std::tm { // inherit "tm" to extend it a little
mytm(int year, int month, int day) : tm{}, current{} {
tm_year = year - 1900;
tm_mon = month - 1;
tm_mday = day;
tm_hour = 12;
current = mktime(this);
}
void add_days(int days) {
current += days * 60 * 60 * 24;
std::memcpy(this, std::localtime(¤t), sizeof(std::tm));
}
std::string strftime(const char* format) {
std::string retval(128, ' '); // a suitable sized buffer
std::size_t len = std::strftime(&retval[0], retval.size(), format, this);
retval.resize(len);
return retval;
}
std::time_t current;
};
int main(void) {
for (int year : {2018, 2019})
for (int month : {1, 11})
for (int day : {1, 31}) {
std::cout <<
mytm(year, month, day)
.strftime("http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=%d-%m-%Y&variation=PT")
<< std::endl;
Sleep(1000);
std::string url =
mytm(year, month, day).strftime("http://www.centrodeinformacao.ren.pt/_layouts/CI.GetExcel/SafeGetExcel.aspx?T=CRG&P=%d-%m-%Y&variation=PT");
std::string file_location =
mytm(year, month, day).strftime("C:/Users/molecoder/Desktop/file_%d_%m_%Y.xls");
// Pass std::string to a function that needs const char* - https://stackoverflow.com/a/347959/5675325
const char * url_final = url.c_str();
const char * file_location_final = file_location.c_str();
downloadFile(url_final, file_location_final);
}
}
Notice that some of the files won't have data and would get the following result
Ocorreu um
erro.REN_CRG_Total::pr_REN_CRG_SelectTotalByDate_Excel::Error occured.
I'm trying to use libcurl (http://curl.haxx.se/libcurl/c/) for downloading data from a web, and store these data in a txt file , and here is my code:
// CLASS SinaStk
size_t save_data(char *buffer, size_t size, size_t nmemb, FILE* userdata){
locale loc = std::locale::global(std::locale("")); //TRY TO OPEN FILE WITH CHINESE
userdata = fopen(fpath.c_str(), "w");
if (userdata == NULL)
printf("File not open!\n");
locale::global(loc);
size_t writelen=size * nmemb;
fwrite(buffer, size, nmemb, userdata);
return writelen;
};
virtual void downloadUrl()
{
CURL* stkCURL=NULL;
CURLcode res;
FILE * fp=NULL;
curl_global_init(CURL_GLOBAL_WIN32);
stkCURL = curl_easy_init();
curl_easy_setopt(stkCURL, CURLOPT_URL,"http://hq.sinajs.cn/list=s_sh000001");
curl_easy_setopt(stkCURL, CURLOPT_WRITEFUNCTION, &SinaStk::save_data);
curl_easy_setopt(stkCURL, CURLOPT_WRITEDATA,fp);
res=curl_easy_perform(stkCURL); //<-STOP!!!!
fclose(fp);
curl_easy_cleanup(stkCURL);
curl_global_cleanup();
return;
};
and when I debug my code, it always stop and then jump to xstring:
size_type size() const _NOEXCEPT
{ // return length of sequence
return (this->_Mysize); // <-STOP!!!
}
0xC0000005: Access violation reading location 0x0000009E
I have no idea about the problem for almost a week. I am upset, I asked people around me and nobody knows why.
Thanks for reading, I am really confused.
=============
Problem is solved! Thanks you guys! now my code is:
//CLASS StkApiInfo
size_t writeData(char* buffer, size_t size, size_t nmemb){
if (stkFile.is_open()){
stkFile.close();
stkFile.clear();
};
fpath = "D:\\Code\\代码\\数据文件\\" + fname + ".txt";
stkFile.open(fpath.c_str(), ios::out);
//if (stkFile.is_open())
cout << buffer<<size<<nmemb;
stkFile << buffer<<endl;
stkFile.close();
stkFile.clear();
return size*nmemb;
};
//CLASS SinaStk : public StkApiInfo
static size_t save_data(char *buffer, size_t size, size_t nmemb, void* userdata){
SinaStk* self = (SinaStk*)userdata;
return self->writeData(buffer, size, nmemb);
};
virtual void downloadUrl()
{
CURL* stkCURL = NULL;
CURLcode res;
curl_global_init(CURL_GLOBAL_WIN32);
stkCURL = curl_easy_init();
if (stkCURL)
{
curl_easy_setopt(stkCURL, CURLOPT_URL, stkUrl.c_str());
curl_easy_setopt(stkCURL, CURLOPT_WRITEFUNCTION, &SinaStk::save_data);
curl_easy_setopt(stkCURL, CURLOPT_WRITEDATA, this);
res = curl_easy_perform(stkCURL);
//if (res != CURLE_OK)
curl_easy_cleanup(stkCURL);
curl_global_cleanup();
}
return;
};
Callback passed with CURLOPT_WRITEFUNCTION argument should be of type write_callback (with exact that signature) and therefore cannot be non-static class method. Usual workaround is to define callback as non-member or static method and pass this as an argument:
static size_t save_data(char *buffer, size_t size, size_t nmemb, void* userdata)
{
SinaStk* self = (SinaStk*) userdata;
return self->doStuff(buffer, size, nmemb);
}
virtual void downloadUrl()
{
//...
curl_easy_setopt(stkCURL, CURLOPT_WRITEFUNCTION, &SinaStk::save_data);
curl_easy_setopt(stkCURL, CURLOPT_WRITEDATA, this);
//...
}
If you need to access additional data (like FILE* in your example) you can either store it as class field or introduce temporary structure that would contain this and additional data fields and pass it's address as callback argument.
I am using http://ubuntuforums.org/showthread.php?t=781021 as a guide for how to write a c++ program that use libcurl to check a website and downloads the text to a c++ app. When I copy the code to my visual studio project, it works with only slight modification. However, I hit a snag trying to convert it to class based files.
I created a .h and .cpp files to run the query. What I didn't realize before was that handle_data was not a typical function, since it is not given any arguments or even () in the midway down Viewer.cpp. This is how the above site did it, and it worked, but I'm at a loss as to why or how to convert it now.
Does anyone know whats going on here or how to fix it?
Thanks!
Viewer.h:
#pragma once
#include <string>
#include "curl.h"
class Viewer
{
public:
Viewer(void);
~Viewer(void);
std::string view(std::string q);
private:
size_t handle_data(void *ptr, size_t size, size_t nmemb, void *stream);
std::string contents;
};
Viewer.cpp:
#include "stdafx.h"
#include "Viewer.h"
#include <iostream>
Viewer::Viewer(void)
{
std::cout << "ViewerCreated!\n";
}
Viewer::~Viewer(void)
{
}
size_t Viewer::handle_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
int numbytes = size*nmemb;
// The data is not null-terminated, so get the last character, and replace
// it with '\0'.
char lastchar = *((char *) ptr + numbytes - 1);
*((char *) ptr + numbytes - 1) = '\0';
contents.append((char *)ptr);
contents.append(1,lastchar);
*((char *) ptr + numbytes - 1) = lastchar; // Might not be necessary.
return size*nmemb;
}
std::string Viewer::view(std::string q)
{
char* url = "www.google.com";
CURL* curl = curl_easy_init();
if(curl)
{
// Tell libcurl the URL
curl_easy_setopt(curl,CURLOPT_URL, url);
// Tell libcurl what function to call when it has data
curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION,handle_data);
// Do it!
CURLcode res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
if (res == 0)
std::cout << contents << std::endl;
else
std::cerr << "Error: " << res << std::endl;
}
return contents;
}
You should not replace the last character of the chunk that you have received. Instead you should force that the buffer containing the appended blocks has a '\0' just after the end of the data. This way it can be printed, but if you are going to store that in a file, you should just write the bytes that you have received.
This has always worked for me:
struct MemoryStruct chunk;
chunk.memory=NULL;
chunk.size = 0;
...
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk);
And the callback:
struct MemoryStruct {
char *memory;
size_t size;
};
static void *myrealloc(void *ptr, size_t size)
{
if(ptr)
return realloc(ptr, size);
else
return malloc(size);
}
static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
{
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)data;
mem->memory = (char *)myrealloc(mem->memory, mem->size + realsize + 1);
if (mem->memory) {
memcpy(&(mem->memory[mem->size]), ptr, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
}
return realsize;
}
handle_data is used as a parameter for curl_easy_setopt. What that means is that you have to program what handle_data does when it receives the next data chunk (providing the data chunk is what curl will do for you).
handle_data is being passed to curl_easy_setopt as a callback function, which it will call when there's data available. Because of that, handle_data must be called as a static function (that is, one not receiving an object pointer, because cURL doesn't have one). However, you're not declaring it as static. Even if you do, you are accessing contents inside it, which is invalid.
I'm trying to save site's source code to vector, where every line of source code is a new vector element, because I only need to use one specific line (number 47) in my program. Any idea how to do this?
Load the data from the URL.
Using cURL:
std::vector<char> LoadFromUrl(const std::string& url)
{
struct Content
{
std::vector<char> data;
static size_t Write(char * data, size_t size, size_t nmemb, void * p)
{
return static_cast<Content*>(p)->WriteImpl(data, size, nmemb);
}
size_t WriteImpl(char* ptr, size_t size, size_t nmemb)
{
data.insert(end(data), ptr, ptr + size * nmemb);
return size * nmemb;
}
};
Content content;
CURL* curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &Content::Write);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_perform(curl);
content.data.push_back('\0');
return content.data;
}
Tokenize the data using strtok or boost tokenizer or your own implementation:
std::vector<std::string> LoadLines(const std::string& url)
{
std::vector<char> content = LoadFromUrl(url);
std::vector<std::string> lines;
for(char* token = strtok(&content.front(), "\n");
token; token = strtok(0, "\n"))
{
lines.push_back(std::string(token));
}
return lines;
}
int main()
{
std::vector<std::string> lines = LoadLines(
"http://stackoverflow.com/questions/10773009/save-sites-source-code-to-vectorstring");
std::copy(begin(lines), end(lines), std::ostream_iterator<std::string>(std::cout, "\n"));
}
I'd like to use the libcurl library to open a remote date file and iterate through it with an istream. I've looked through the nice example in this thread but it writes the remote file to a local file. Instead I'd like to have the remote reads be pushed to an istream for subsequent programmatic manipulation. Is this possible? I would greatly appreciate help.
Best,
Aaron
Boost's IO Stream might be a better solution than STL's own stream. At least it is much simpler to create a boost stream. From boost's own docs:
#include <curl/curl.h>
#include <boost/iostreams/stream.hpp>
class CURLDevice
{
private:
CURL* handle;
public:
typedef char char_type;
typedef boost::iostreams::source_tag category;
CURLDevice()
{
handle = curl_easy_init();
}
CURLDevice(const std::string &url)
{
handle = curl_easy_init();
open( url );
}
~CURLDevice()
{
curl_easy_cleanup(handle);
}
void open(const std::string &url)
{
curl_easy_setopt(handle, CURLOPT_URL, url.c_str());
curl_easy_setopt(handle, CURLOPT_CONNECT_ONLY, 1);
curl_easy_perform(handle);
}
std::streamsize read(char* s, std::streamsize n)
{
size_t read;
CURLcode ret = curl_easy_recv(handle, s, n, &read);
if ( ret == CURLE_OK || ret == CURLE_AGAIN )
return read;
else
return -1;
}
};
typedef boost::iostreams::stream<CURLDevice> CURLStream;
int main(int argc, char **argv)
{
curl_global_init(CURL_GLOBAL_ALL);
{
CURLStream stream("http://google.com");
char buffer[256];
int sz;
do
{
sz = 256;
stream.read( buffer, sz );
sz = stream.gcount();
std::cout.write( buffer, sz );
}
while( sz > 0 );
}
curl_global_cleanup();
return 0;
}
Note: when I run the code above I get a segfault in CURL, this appears to be because I don't know exactly how to use curl itself.