I am trying to read automatically some information from investing.com using QNetworkAccessManager. I can read from other sites but this site gives some webmaster tools which I want to access.
https://www.investing.com/webmaster-tools/
I use this query which works in a browser.
Here is my request code
class InvestingAPI: public QObject
{
Q_OBJECT
public:
InvestingAPI();
QueryTechnicals(QString Symbol, int TF1Minites);
signals:
// void NewTechnicalSummary(int Timeframe, QString Symbol, QString Summary);
private slots:
void onData(QNetworkReply *reply);
private:
QNetworkAccessManager qnam ;
};
InvestingAPI::InvestingAPI()
{
connect(&qnam,SIGNAL(finished(QNetworkReply*)),this,SLOT(onData(QNetworkReply*));
connect(&qnam,SIGNAL(encrypted(QNetworkReply*)),this,SLOT(onData(QNetworkReply*))
);
}
InvestingAPI::QueryTechnicals(QString Symbol, int TF1Minites)
{
QString Query;
Query = "http://ssltsw.forexprostools.com/index.php?timeframe=300&lang=1&forex=1&commodities=8830,8836,8831,8849,8833,8862,8832&indices=175,166,172,27,179,170,174&stocks=334,345,346,347,348,349,350&tabs=1,2,3,4%22%20width=%22317%22%20height=%22467%22%3E%3C/iframe%3E%3Cdiv%20class=%22poweredBy%22%20style=%22font-family:arial,helvetica,sans-serif;%20direction:ltr;%22%3E%3Cspan%20style=%22font-size:%2011px;color:%20&selectedTabId=QBS_1";
QNetworkRequest Request;
Request.setSslConfiguration(QSslConfiguration::defaultConfiguration());
connect(&qnam,SIGNAL(finished(QNetworkReply*)),this,SLOT(onData(QNetworkReply*)));
Request.setUrl(QUrl(Query));
Request.setRawHeader("User-Agent", "MyOwnBrowser 1.0");
qnam.get(Request);
}
And I have event
void InvestingAPI::onData(QNetworkReply *reply){
// find data type
// decode and return data to caller
if(reply->error() != QNetworkReply::NoError){
qDebug() << "Error";
qDebug() << reply->errorString();
}
QString html = QString::fromUtf8(reply->readAll());
qDebug() << html;
QString SubData;
}
I do not get an error but I get an empty string rather than the html response.
Please help as I have no idea why this is not working here but is working in the browser.
By default Qt Network does not handle redirects like other tools, so that is why you get an empty data (if you check the "Location" header you will see the redirected url). In this case it is to enable the redirection:
Request.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
Related
I am going to use QNetworkAccessManager to make requests to HTTP server by my mobile app to the server. The question is, how do you link custom data to each request ? I tried to subclass QNetworkReply but I found out that I have to implement virtual methods close() and isSequential() but I don't know what those should return so I am afraid I am going to break network request functionality.
For example, when my app does the log in procedure, it has to store the email address of the account:
class MyApp : public QObject
{
Q_OBJECT
private:
QNetworkRequest request;
QNetworkReply *reply;
QNetworkAccessManager *manager;
...
}
void MyApp::do_log_in(QString email, QString password) {
QString s;
someobject.email=email; // <-- I have to store email address before sending request to server, but where do I store it?
s.append("http://myapp.com/do-auth.php?email=");
s.append(QUrl::toPercentEncoding(email));
s.append("&password=");
s.append(QUrl::toPercentEncoding(password));
connect(manager,SIGNAL(finished(QNetworkReply*)),this,SLOT(login_finished(QNetworkReply*)));
request.setUrl(QUrl(s));
manager->get(request);
}
void MyApp::login_finished(QNetworkReply *rep) {
DepservReply *reply;
QString email;
....
email= ...... // <-- I need to get the email address from QNetworkReply object somehow
///my code here handling server reply
....
}
So, how do I implement storage and retrieval of email in my case, what classes should I subclass and what methods should I re-implement ?
You can leverage the dynamic property system available in each QObject and hold the data in the reply:
// https://github.com/KubaO/stackoverflown/tree/master/questions/network-reply-tracking-40707025
#include <QtNetwork>
class MyCtl : public QObject
{
Q_OBJECT
QNetworkAccessManager manager{this};
// ...
void reply_finished(QNetworkReply *reply);
public:
MyCtl(QObject *parent = nullptr);
void do_log_in(const QString &email, const QString &password);
};
static const char kAuthGetSalt[] = "req_auth-get-salt";
static const char kDoAuth[] = "req_do-auth";
static const char kEmail[] = "req_email";
static const char kPassword[] = "req_password";
static const auto authGetSaltUrl = QStringLiteral("https://myapp.com/auth-get-salt.php?email=%1");
static const auto doAuthUrl = QStringLiteral("https://myapp.com/do-auth.php?email=%1&passwordHash=%2");
MyCtl::MyCtl(QObject *parent) : QObject{parent}
{
connect(&manager, &QNetworkAccessManager::finished, this, &MyCtl::reply_finished);
}
void MyCtl::do_log_in(const QString &email, const QString &password) {
auto url = authGetSaltUrl.arg(email);
auto reply = manager.get(QNetworkRequest{url});
reply->setProperty(kAuthGetSalt, true);
reply->setProperty(kEmail, email);
reply->setProperty(kPassword, password);
}
void MyCtl::reply_finished(QNetworkReply *reply) {
if (!reply->property(kAuthGetSalt).isNull()) {
reply->deleteLater(); // let's not leak the reply
if (reply->error() == QNetworkReply::NoError) {
auto salt = reply->readAll();
auto email = reply->property(kEmail).toString();
auto password = reply->property(kPassword).toString();
Q_ASSERT(!password.isEmpty() && !email.isEmpty());
QCryptographicHash hasher{QCryptographicHash::Sha1};
hasher.addData(salt); // the server must hash the same way
hasher.addData("----");
hasher.addData(password.toUtf8());
auto hash = hasher.result().toBase64(QByteArray::Base64UrlEncoding);
auto url = doAuthUrl.arg(email).arg(QString::fromLatin1(hash));
auto reply = manager.get(QNetworkRequest{url});
reply->setProperty(kDoAuth, true);
reply->setProperty(kEmail, email);
}
}
else if (!reply->property(kDoAuth).isNull()) {
if (reply->error() == QNetworkReply::NoError) {
auto email = reply->property(kEmail).toString();
// ...
}
}
}
Use a constant for a property name to avoid typos by letting the compiler check that you're using a valid identifier.
The example above rectifies the following critical safety issues in your code:
Sending security credentials over a clear connection: use https://, not http://.
Sending a password in cleartext: instead, send a salted hash of it. Your server should generate a random salt for each account when the accounts are created. Existing accounts can be left unsalted, but they should acquire a salt as soon as the user changes the password.
Also note that a QString to QUrl conversion will automatically percent-encode the string, so doing it explicitly is unnecessary.
In this case email is part of the request's URL so you could just extract it from there (the QNetworkReply has access to the QNetworkRequest it is handling, see QNetworkReply::request()).
You an also store more or less any kind of data as a dynamic property because QNetworkReply is a QObject derived class, see QObject::setProperty().
You can subclass QNAM to get more control of it.
network.h
class QNAMNetwork : public QNetworkAccessManager
{
Q_OBJECT
public:
explicit QNAMNetwork(QObject *parent = 0);
~QNAMNetwork();
inline void insertUserValue(const QString & key, const QString & value){this->m_user_values.insert(key,value);}
inline QString getUserValue(const QString & key){return this->m_user_values.value(key);}
signals:
void requestFinished(ExNetwork *, QNetworkReply *);
private slots:
void _sslErrors(QNetworkReply *, const QList<QSslError> &);
void _finished(QNetworkReply *);
private:
QMap<QString, QString> m_user_values;
};
network.cpp
QNAMNetwork::QNAMNetwork(QObject *parent):QNetworkAccessManager(parent)
{
connect(this, &QNAMNetwork::sslErrors, this, &QNAMNetwork::_sslErrors);
connect(this, &QNAMNetwork::finished, this, &QNAMNetwork::_finished);
}
QNAMNetwork::~QNAMNetwork()
{
//qDebug() << __FUNCTION__ << QString::number((long)this,16);
}
void QNAMNetwork::_sslErrors(QNetworkReply * reply, const QList<QSslError> & errors)
{
reply->ignoreSslErrors(errors);
}
void QNAMNetwork::_finished(QNetworkReply * reply)
{
emit requestFinished(this, reply);
}
usecase:
QNAMNetwork * network = new QNAMNetwork(this);
network->insertUserValue("email","yourmail#mail.com");
connect(network, SIGNAL(requestFinished(QNAMNetwork*,QNetworkReply*)), this, SLOT(requestFinished(QNAMNetwork*,QNetworkReply*)));
QNetworkRequest req(QUrl::fromUserInput(query)); //get url
network->get(req);
...
void YourClass::requestFinished(QNAMNetwork * net, QNetworkReply * rep)
{
QString email = net->getUserValue("email");
net->deleteLater();
rep->deleteLater();
}
In QWebView it was possible to set a QNetworkCookieJar via QNetworkAccessManager.
QNetworkAccessManager *nam = new QNetworkAccessManager();
nam->setCookieJar(cookieJar);
webView->page()->setNetworkAccessManager(nam);
This was working like a charm.
How can I set a QNetworkCookieJar in new QWebEngine class introduced in Qt5.4?
I found a solution, to share Cookies with QWebEngine and QNetworkAccesManager by using QWebEngineCookieStore. Subclass QNetworkCookieJar:
class CookieWebEngine : public QNetworkCookieJar
{
.....
protected:
// Reimplement this functions to work with your _cookies list;
bool insertCookie(const QNetworkCookie &cookie);
bool deleteCookie(const QNetworkCookie &cookie);
bool updateCookie(const QNetworkCookie &cookie);
bool validateCookie(const QNetworkCookie &cookie, const QUrl &url) const;
private:
// Save Chromium Cookies
QWebEngineCookieStore *_store;
// Save available cookies
QList<QNetworkCookie> _cookies;
}
Now, lets implement a function to load/save cookies in a file:
// Load Chromium Cookies
void CookieWebEngine::loadStore() {
// Save cookies profile shared
QWebEngineProfile::defaultProfile()->setPersistentCookiesPolicy(QWebEngineProfile::ForcePersistentCookies);
_store = WebEngineProfile::defaultProfile()->cookieStore();
connect(_store, &QWebEngineCookieStore::cookieAdded, this, &CookieWebEngine::handleCookieAdded);
_store->loadAllCookies();
}
// Load/Save cookies in arraylist in a file
void CookieWebEngine::load() {
// load cookies and exceptions
qRegisterMetaTypeStreamOperators<QList<QNetworkCookie> >("QList<QNetworkCookie>");
const QString location = cookiesDirectory() + COOKIES_FILE;
QSettings cookieSettings(location, QSettings::IniFormat);
_cookies = qvariant_cast<QList<QNetworkCookie> >(cookieSettings.value(QLatin1String("cookies")));
setAllCookies(_cookies);
// Now user iterate and add it to chromium
for (auto cookie : _cookies) {
_store->setCookie(cookie);
}
cookieSettings.sync();
}
void CookieWebEngine::save()
{
QString directory = cookiesDirectory();
if (!QFile::exists(directory)) {
QDir dir;
dir.mkpath(directory);
}
const QString location = directory + COOKIES_FILE;
QSettings cookieSettings(location, QSettings::IniFormat);
cookieSettings.setValue(QLatin1String("cookies"), QVariant::fromValue<QList<QNetworkCookie>>(_cookies));
cookieSettings.sync();
}
Now, just connect and handle cookies loaded from the webview:
void CookieWebEngine::handleCookieAdded(const QNetworkCookie &cookie)
{
if (insertCookie(cookie)) {
qDebug() << "Handle cookie " << cookie;
}
}
Its working well for me. Now, i use chromium to sign in. After, i save my session cookie in the customized cookiejar and I use it in my customized QNetworkAccesManager.
I want to intercept, inspect and (if needed) reject AJAX-Requests based on the Fingerprint of the SSL-Certificate. I use the QNetworkAccessManager::createRequest(...) function to issue requests. Everything works fine when I use QWebFrame::load(...). Even the content which is loaded within the request (like .css or .js files) emit signals. Unfortunately no AJAX-Requests emits any Signals. I know that the Signals are connected to the very same slots (for "normal" as well as AJAX-Requests) within MyNetworkAccessManager::createRequest(...) function.
QNetworkReply *reply = QNetworkAccessManager::createRequest(op, req, outgoingData);
connect(reply, SIGNAL(readyRead()), this, SLOT(handleStarted()));
connect(reply, SIGNAL(sslErrors(const QList<QSslError> &)), this, SLOT(handleSslErrors(const QList<QSslError> &)));
connect(reply, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(handleNetworkError()));
Why are AJAX Requests so different? Where can I access them?
From what I can tell, AJAX requests do emit the finished signal on QNetworkAccessManager. You need connect to the instance of QNetworkAccessManager on your QWebPage instance:
QWebPage *page = ui->webView->page();
QNetworkAccessManager *nam = page->networkAccessManager();
connect(nam, SIGNAL(finished(QNetworkReply*)), this, SLOT(replyFinished(QNetworkReply*)));
QFile file;
file.setFileName(":/js/jquery-2.1.1.min.js"); // jQuery is loaded as a resource
file.open(QIODevice::ReadOnly);
QString jQuery = file.readAll();
file.close();
ui->webView->load(QUrl("about:blank"));
QWebFrame *frame = m_page->mainFrame();
frame->evaluateJavaScript(jQuery); // load jQuery
// check that jQuery is loaded
frame->evaluateJavaScript("$(document).ready(function() { alert('jQuery loaded!'); });");
// do an AJAX GET
frame->evaluateJavaScript("$.ajax({"
"url: 'http://www.json-generator.com/api/json/get/cqkXBAEoQy?indent=2',"
"method: 'GET',"
"dataType: 'json'"
"}).done(function (data) {"
"for (var i = 0; i < data.length; i++) {"
"alert(data[i].name);"
"}"
"}).error(function (data) { alert('AJAX error'); });");
Then you can monitor replies in the replyFinished slot like so:
void MainWindow::replyFinished(QNetworkReply *reply)
{
QByteArray bytes = reply->readAll();
QString str = QString::fromUtf8(bytes.data(), bytes.size());
QString replyUrl = reply->url().toString();
int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
qDebug() << statusCode;
qDebug() << replyUrl;
qDebug() << str;
}
I have noticed that jQuery AJAX's done promise doesn't seem to execute when you do anything with the QNetworkReply, but you can see that the request actually finishes in the debug console.
See my GitHub repository to try out the above code: https://github.com/pcmantinker/QtWebkitAJAX
As far as blocking connections based on SSL certificates, you'd have to subclass QNetworkAccessManager and override QNetworkAccessManager::createRequest. Something like this could work:
QNetworkReply *CustomQNetworkAccessManager::createRequest(Operation op, const QNetworkRequest& request, QIODevice* outgoingData)
{
QNetworkRequest req(request);
QNetworkReply *reply = QNetworkAccessManager::createRequest(op, req, outgoingData);
QSslConfiguration *sslConfig = reply->sslConfiguration();
QList<QSslCertificate> sslCaCerts = sslConfig->caCertificates();
// do something with sslCaCerts
return reply;
}
I need to login to a website to retrieve the source code of a page. How would I do this using Qt?
I'm not familiar with how QUrl and QNetworkAcessManager work, but I was able to write code that would allow me to download the source code for any page not behind a login form.
This is what I have so far. I end up just downloading the source for the redirect page:
test = new QNetworkAccessManager(this);
QUrl URL = QUrl("http://website.com/page");
URL.setUserName("user");
URL.setPassword("password");
test->get(QNetworkRequest(URL));
Edit:
QByteArray loginData("username=user&password=password");
QNetworkRequest request(QUrl("http://website.com/login/index.php"));
manager->post(request,loginData);
QUrl URL = QUrl("http://website.com/mod/resource/view.php?id=114198");
manager->get(QNetworkRequest(URL));
I am still retrieving a 303 reply.
The page is on Moodle, which uses a HTTP POST login form.
I've also tried with a different site. POST works but I get the source code of the login page with the login form filled out. Not sure how to get the page after logging in.
This was my solution, for logging into a particular website and retrieving a file. I was getting redirected multiple times before reaching the destination. I test HTTP status codes until I get the final reply. I also test to see if the final URL is the one I requested and not some home page behind the login page. This is because I am downloading a file.
QByteArray loginData;
loginData.append("username="+(ui->lineEdit_2->text())+"&password="+(ui->lineEdit_3->text())+"&action=login");
cookiesHandler* test = new cookiesHandler(this);
QUrl request("http://website.com");
test->sendPostRequest(request, loginData);
Cookies Handler Class
class cookiesHandler: public QObject{
Q_OBJECT
public:
cookiesHandler(QObject *parent = 0) : QObject(parent){
mManager = new QNetworkAccessManager(this);
mManager->setCookieJar(new QNetworkCookieJar(this));
connect(mManager, SIGNAL(finished(QNetworkReply*)), SLOT(replyFinished(QNetworkReply*)));
}
void sendPostRequest(const QUrl &url, const QByteArray &data){
mUrl = url;
login = data;
QNetworkRequest r(mUrl);
mManager->post(r, data);
}
void sendGetRequest(const QUrl &url)
{
mUrl = url;
test = mUrl;
QNetworkRequest r(mUrl);
mManager->get(r);
}
virtual ~cookiesHandler(){}
private slots:
void replyFinished(QNetworkReply *reply){
if (reply->error() != QNetworkReply::NoError){
qWarning() << "ERROR:" << reply->errorString();
return;
}
//Cookies//
QList<QNetworkCookie> cookies = mManager->cookieJar()->cookiesForUrl(mUrl);
qDebug() << "COOKIES for" << mUrl.host() << cookies;
QString str;
QDebug dStream(&str);
dStream << mUrl.host() << cookies;
//End Cookies//
int v = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (v >= 200 && v < 300) // Success
{
getFile(reply);
// Here we got the final reply
return;
}
else if (v >= 300 && v < 400) // Redirection
{
/* Use Cookies for Login */
qDebug() << "REDIRECTING";
rUrl = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
if(rUrl != mUrl)
{
mManager->post(QNetworkRequest(rUrl),login);
return;}
out << QString("redirected: " + rUrl.toEncoded()) << endl;
QNetworkRequest r(mUrl);
QVariant var;
var.setValue(cookies);
r.setHeader(QNetworkRequest::CookieHeader, var);
mManager->get(r);
return;
}
}
I am trying to save cookies that are produced by my app to disk location such as C:\Users\Username\AppData\Local\MyCompany\MyApp. I have implemented a webview and have pretty much finished coding my simple browser the final thing to do is save cookies.
I am can qDebug() the cookies I get from the webapp and they show the cookies are formed correctly but I am a)unsure where to go from there and b) not 100% sure on how to make a subclass of the cookiejar class?
Below I create my cookiejar object in my MainWindow constructor
view = new QWebView(this);
jar = new QNetworkCookieJar;
view->page()->networkAccessManager()->setCookieJar(jar);
And in my replyfinished slot I can see the cookie contained in the reply and I attempt to save it but nothing happens and I receive no run time errors. There isn't a great deal of stuff out there on this and have seen a few posts where the instruction was to make a subclass QNetworkCookieJar but have not made a subclass in Qt/C++ before.
Is there a simple way to store cookies, I am not looking for anything fancy. The cookies just make sure some check boxes are ticked on the login page.
// SLOT that accepts the read data from the webpage
void MainWindow::slotReplyFinished(QNetworkReply *reply){
if(reply->isFinished()){
QVariant variantCookies = reply->header(QNetworkRequest::SetCookieHeader);
QList<QNetworkCookie> cookies = qvariant_cast<QList<QNetworkCookie> >(variantCookies);
qDebug() << "Cookies reply: " << cookies;
QNetworkCookie cookie; //Create a cookie
jar = new QNetworkCookieJar;
//view->page()->networkAccessManager()->setCookieJar(jar);
jar->setCookiesFromUrl(cookies, reply->request().url());
//qDebug() << "Saved cookies: " << jar->getAllCookies();
}
qDebug() << "Network reply: " << reply->errorString() << reply->error() << reply->request().url();
}
You will need to subclass QNetworkCookieJar and in that class you should implement your own persistent storage.
class MyNetworkCookieJar : public QNetworkCookieJar {
public:
bool saveCookiesToDisk() {
// .. my implementation
return true; // if i did
}
bool loadCookiesFromDisk() {
// .. load from disk
return false; // if unable to.
}
}
The sample application from Qt project implements a persistent cookie store, it could be a good starting point for you: http://qt.gitorious.org/qt/qt/trees/4.8/demos/browser
look at cookiejar.h and cookiejar.cpp
Base of qt example, http://qt.gitorious.org/qt/qt/trees/4.8/demos/browser, i wrote this class that save and use one cookie for me. Perhaps it helps you too. Note that it just save one cookie and not list of cookies.
#include "cookiejar.h"
CookieJar::CookieJar(QObject *parent)
: QNetworkCookieJar(parent)
, m_loaded(false)
{
}
void CookieJar::load()
{
if (m_loaded)
return;
QSettings settings;
settings.beginGroup(QLatin1String("cookies"));
QList<QNetworkCookie> savedCookies = QNetworkCookie::parseCookies(settings.value("cookies").toByteArray());
for (int j = 0; j < savedCookies.count(); j++)
insertCookie(savedCookies.at(j));
m_loaded = true;
emit cookiesChanged();
}
void CookieJar::save()
{
if (!m_loaded)
return;
QList<QNetworkCookie> cookies = allCookies();
QSettings settings;
settings.beginGroup(QLatin1String("cookies"));
settings.setValue("cookies", cookies[0].toRawForm());
}
QList<QNetworkCookie> CookieJar::cookiesForUrl(const QUrl &url) const
{
// This function is called by the default QNetworkAccessManager::createRequest(),
// which adds the cookies returned by this function to the request being sent.
CookieJar *that = const_cast<CookieJar*>(this);
if (!m_loaded)
that->load();
return QNetworkCookieJar::cookiesForUrl(url);
}
bool CookieJar::setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url)
{
if (!m_loaded)
load();
QNetworkCookieJar::setCookiesFromUrl(cookieList, url);
save(); //Save cookie permanently in setting file.
emit cookiesChanged();
return true;
}
#Musa's answer is good but it only saves one cookie. I recommend using the Qt folk's implementation from the old qmlviewer located here: http://code.qt.io/cgit/qt/qt.git/tree/tools/qml/qmlruntime.cpp?h=4.7#n438
Here's the code:
class PersistentCookieJar : public QNetworkCookieJar {
public:
PersistentCookieJar(QObject *parent) : QNetworkCookieJar(parent) { load(); }
~PersistentCookieJar() { save(); }
virtual QList<QNetworkCookie> cookiesForUrl(const QUrl &url) const
{
QMutexLocker lock(&mutex);
return QNetworkCookieJar::cookiesForUrl(url);
}
virtual bool setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url)
{
QMutexLocker lock(&mutex);
return QNetworkCookieJar::setCookiesFromUrl(cookieList, url);
}
private:
void save()
{
QMutexLocker lock(&mutex);
QList<QNetworkCookie> list = allCookies();
QByteArray data;
foreach (QNetworkCookie cookie, list) {
if (!cookie.isSessionCookie()) {
data.append(cookie.toRawForm());
data.append("\n");
}
}
QSettings settings;
settings.setValue("Cookies",data);
}
void load()
{
QMutexLocker lock(&mutex);
QSettings settings;
QByteArray data = settings.value("Cookies").toByteArray();
setAllCookies(QNetworkCookie::parseCookies(data));
}
mutable QMutex mutex;
};