I need to login to a website to retrieve the source code of a page. How would I do this using Qt?
I'm not familiar with how QUrl and QNetworkAcessManager work, but I was able to write code that would allow me to download the source code for any page not behind a login form.
This is what I have so far. I end up just downloading the source for the redirect page:
test = new QNetworkAccessManager(this);
QUrl URL = QUrl("http://website.com/page");
URL.setUserName("user");
URL.setPassword("password");
test->get(QNetworkRequest(URL));
Edit:
QByteArray loginData("username=user&password=password");
QNetworkRequest request(QUrl("http://website.com/login/index.php"));
manager->post(request,loginData);
QUrl URL = QUrl("http://website.com/mod/resource/view.php?id=114198");
manager->get(QNetworkRequest(URL));
I am still retrieving a 303 reply.
The page is on Moodle, which uses a HTTP POST login form.
I've also tried with a different site. POST works but I get the source code of the login page with the login form filled out. Not sure how to get the page after logging in.
This was my solution, for logging into a particular website and retrieving a file. I was getting redirected multiple times before reaching the destination. I test HTTP status codes until I get the final reply. I also test to see if the final URL is the one I requested and not some home page behind the login page. This is because I am downloading a file.
QByteArray loginData;
loginData.append("username="+(ui->lineEdit_2->text())+"&password="+(ui->lineEdit_3->text())+"&action=login");
cookiesHandler* test = new cookiesHandler(this);
QUrl request("http://website.com");
test->sendPostRequest(request, loginData);
Cookies Handler Class
class cookiesHandler: public QObject{
Q_OBJECT
public:
cookiesHandler(QObject *parent = 0) : QObject(parent){
mManager = new QNetworkAccessManager(this);
mManager->setCookieJar(new QNetworkCookieJar(this));
connect(mManager, SIGNAL(finished(QNetworkReply*)), SLOT(replyFinished(QNetworkReply*)));
}
void sendPostRequest(const QUrl &url, const QByteArray &data){
mUrl = url;
login = data;
QNetworkRequest r(mUrl);
mManager->post(r, data);
}
void sendGetRequest(const QUrl &url)
{
mUrl = url;
test = mUrl;
QNetworkRequest r(mUrl);
mManager->get(r);
}
virtual ~cookiesHandler(){}
private slots:
void replyFinished(QNetworkReply *reply){
if (reply->error() != QNetworkReply::NoError){
qWarning() << "ERROR:" << reply->errorString();
return;
}
//Cookies//
QList<QNetworkCookie> cookies = mManager->cookieJar()->cookiesForUrl(mUrl);
qDebug() << "COOKIES for" << mUrl.host() << cookies;
QString str;
QDebug dStream(&str);
dStream << mUrl.host() << cookies;
//End Cookies//
int v = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (v >= 200 && v < 300) // Success
{
getFile(reply);
// Here we got the final reply
return;
}
else if (v >= 300 && v < 400) // Redirection
{
/* Use Cookies for Login */
qDebug() << "REDIRECTING";
rUrl = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
if(rUrl != mUrl)
{
mManager->post(QNetworkRequest(rUrl),login);
return;}
out << QString("redirected: " + rUrl.toEncoded()) << endl;
QNetworkRequest r(mUrl);
QVariant var;
var.setValue(cookies);
r.setHeader(QNetworkRequest::CookieHeader, var);
mManager->get(r);
return;
}
}
Related
I would like to create an HTTP request and retrieve in the response some variables create during the request.
However, QNetworkAccessManager reponse is asynchronous and use SIGNAL/SLOT functionality, so variable are no more accessible.
Here is an example :
void makeRequest()
{
manager = new QNetworkAccessManager(this);
connect(manager, SIGNAL(finished(QNetworkReply*)),
this, SLOT(replyFinished(QNetworkReply*)));
// here is the datas I want to get in the reply
// objectdata and objectdata2 are created just before the http request
MyClass objectdata("randomseed");
MyClass2 objectdata("randomseed") ;
QUrl websiteurl = objectdata.getUrl();
manager->get(QNetworkRequest(websiteurl));
}
void replyFinished (QNetworkReply *reply)
{
if(reply->error())
{
qDebug() << "ERROR!";
qDebug() << reply->errorString();
}
else
{
QByteArray dataHttp = reply->readAll();
// How can I get here objectdata and objectdata2 ?
// I would like to do something like that this->dataSuccess(objectdata,objectdata2,dataHttp);
}
reply->deleteLater();
}
In a synchronous system this problematic doesn't exist.
Is there a workaround to this problematic ?
One way to do it is to capture the locals in a lambda instead of using a separate function for the slot.
void makeRequest()
{
manager = new QNetworkAccessManager(this);
// here is the datas I want to get in the reply
// objectdata and objectdata2 are created just before the http request
MyClass objectdata("randomseed");
MyClass2 objectdata("randomseed") ;
connect(manager, &QNetworkAccessManager::finished, [this, objectdata, objectdata2](QNetworkReply *reply)
{
if(reply->error())
{
qDebug() << "ERROR!";
qDebug() << reply->errorString();
}
else
{
QByteArray dataHttp = reply->readAll();
dataSuccess(objectdata,objectdata2,dataHttp);
}
reply->deleteLater();
}
QUrl websiteurl = objectdata.getUrl();
manager->get(QNetworkRequest(websiteurl));
}
Another way is to add extra parameters as an attribute in the QNetworkRequest.
// Save off the data in the request
QNetworkRequest req(websiteurl);
req.setAttribute(QNetworkRequest::User, QVariant::fromValue<MyClass>(objectdata));
req.setAttribute(QNetworkRequest::User + 1, QVariant::fromValue<MyClass2>(objectdata2));
manager->get(req);
...
// Retrieve the data from the reply
auto data1 = reply->request().attribute(QNetworkRequest::User).value<MyClass>();
auto data2 = reply->request().attribute(QNetworkRequest::User + 1).value<MyClass2>();
I am trying to read automatically some information from investing.com using QNetworkAccessManager. I can read from other sites but this site gives some webmaster tools which I want to access.
https://www.investing.com/webmaster-tools/
I use this query which works in a browser.
Here is my request code
class InvestingAPI: public QObject
{
Q_OBJECT
public:
InvestingAPI();
QueryTechnicals(QString Symbol, int TF1Minites);
signals:
// void NewTechnicalSummary(int Timeframe, QString Symbol, QString Summary);
private slots:
void onData(QNetworkReply *reply);
private:
QNetworkAccessManager qnam ;
};
InvestingAPI::InvestingAPI()
{
connect(&qnam,SIGNAL(finished(QNetworkReply*)),this,SLOT(onData(QNetworkReply*));
connect(&qnam,SIGNAL(encrypted(QNetworkReply*)),this,SLOT(onData(QNetworkReply*))
);
}
InvestingAPI::QueryTechnicals(QString Symbol, int TF1Minites)
{
QString Query;
Query = "http://ssltsw.forexprostools.com/index.php?timeframe=300&lang=1&forex=1&commodities=8830,8836,8831,8849,8833,8862,8832&indices=175,166,172,27,179,170,174&stocks=334,345,346,347,348,349,350&tabs=1,2,3,4%22%20width=%22317%22%20height=%22467%22%3E%3C/iframe%3E%3Cdiv%20class=%22poweredBy%22%20style=%22font-family:arial,helvetica,sans-serif;%20direction:ltr;%22%3E%3Cspan%20style=%22font-size:%2011px;color:%20&selectedTabId=QBS_1";
QNetworkRequest Request;
Request.setSslConfiguration(QSslConfiguration::defaultConfiguration());
connect(&qnam,SIGNAL(finished(QNetworkReply*)),this,SLOT(onData(QNetworkReply*)));
Request.setUrl(QUrl(Query));
Request.setRawHeader("User-Agent", "MyOwnBrowser 1.0");
qnam.get(Request);
}
And I have event
void InvestingAPI::onData(QNetworkReply *reply){
// find data type
// decode and return data to caller
if(reply->error() != QNetworkReply::NoError){
qDebug() << "Error";
qDebug() << reply->errorString();
}
QString html = QString::fromUtf8(reply->readAll());
qDebug() << html;
QString SubData;
}
I do not get an error but I get an empty string rather than the html response.
Please help as I have no idea why this is not working here but is working in the browser.
By default Qt Network does not handle redirects like other tools, so that is why you get an empty data (if you check the "Location" header you will see the redirected url). In this case it is to enable the redirection:
Request.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
A user using QWebEngineView in my application fills some form. This form uses post method to submit data to server. How can I get params from user's body request?
I've found such thing as QWebEngineUrlRequestInterceptor, but it works only for urls.
You can use QWebEnginePage::acceptNavigationRequest.
Whenever a form is submitted, you can get the contents of input by using JavaScript and then accept the request to proceed as usual.
Like Anmol Gautam said, you need to reimplement QWebEnginePage::acceptNavigationRequest function and get needed data using JavaScript.
Here is an example how to do it:
mywebpage.h
#include <QWebEnginePage>
class MyWebPage : public QWebEnginePage
{
Q_OBJECT
public:
explicit MyWebPage(QWebEngineProfile *profile = Q_NULLPTR, QObject *parent = Q_NULLPTR);
protected:
bool acceptNavigationRequest(const QUrl & url, QWebEnginePage::NavigationType type, bool isMainFrame);
}
mywebpage.cpp
MyWebPage::MyWebPage(QWebEngineProfile *profile, QObject *parent):QWebEnginePage(profile, parent),
{
//...
}
bool MyWebPage::acceptNavigationRequest(const QUrl & url, QWebEnginePage::NavigationType type, bool isMainFrame)
{
if(type == QWebEnginePage::NavigationTypeFormSubmitted)
{
qDebug() << "[FORMS] Submitted" << url.toString();
QString jsform = "function getformsvals()"
"{var result;"
"for(var i = 0; i < document.forms.length; i++){"
"for(var x = 0; x < document.forms[i].length; x++){"
"result += document.forms[i].elements[x].name + \" = \" +document.forms[i].elements[x].value;"
"}}"
"return result;} getformsvals();";
this->runJavaScript(jsform, [](const QVariant &result){ qDebug() << "[FORMS] found: " << result; });
}
return true;
}
use QWebEngineView::setPage to set your WebPage subclass to WebView before you call WebViews load function.
Here is a link for more info about HTML DOM forms Collection
So I have read the documentation at the following link https://github.com/reddit-archive/reddit/wiki/OAuth2. I am trying to retrieve an access token for my application which only requires an Application Only OAuth since it does not require the user to insert their credentials. I have followed the instructions on the page mentioned, but I am unable to retrieve the access token and I always get:
"{\"message\": \"Unauthorized\", \"error\": 401}"
Here is my code:
#include "reddit.h"
#include <QtNetwork>
#include <QUuid>
const QString GRANT_URL = "https://oauth.reddit.com/grants/installed_client";
const QString ACCESS_TOKEN_URL = "https://www.reddit.com/api/v1/access_token";
const QByteArray CLIENT_IDENTIFIER = "MYID";
Reddit::Reddit(QObject *parent) : QObject(parent)
{
mDeviceID = "DO_NOT_TRACK_THIS_DEVICE";
mAuthHeader = "Basic " + CLIENT_IDENTIFIER.toBase64();
}
void Reddit::getAccessToken()
{
auto netManager = new QNetworkAccessManager(this);
QUrl requestUrl = buildAccessTokenUrl();
QNetworkRequest netRequest(requestUrl);
netRequest.setHeader(QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded");
netRequest.setRawHeader("Authorization", mAuthHeader);
auto reply = netManager->post(netRequest, requestUrl.query(QUrl::FullyEncoded).toUtf8());
connect(reply, &QNetworkReply::finished, this, &Reddit::accessTokenRequestFinished);
}
void Reddit::accessTokenRequestFinished()
{
auto reply = qobject_cast<QNetworkReply*>(sender());
qDebug() << reply->readAll();
reply->deleteLater();
}
QUrl Reddit::buildAccessTokenUrl()
{
QUrl url(ACCESS_TOKEN_URL);
QUrlQuery urlQuery;
urlQuery.addQueryItem("grant_type", GRANT_URL);
urlQuery.addQueryItem("device_id", mDeviceID);
url.setQuery(urlQuery);
return url;
}
I have registerd my application at https://ssl.reddit.com/prefs/apps/ using the "installed" type option.
Ok so I found the problem. I didn't read the 'Basic' HTTP Authentication Scheme and forgot a : in my authorization header which I modified to:
mAuthHeader = "Basic " + (CLIENT_IDENTIFIER + ":").toBase64();
I am trying to save cookies that are produced by my app to disk location such as C:\Users\Username\AppData\Local\MyCompany\MyApp. I have implemented a webview and have pretty much finished coding my simple browser the final thing to do is save cookies.
I am can qDebug() the cookies I get from the webapp and they show the cookies are formed correctly but I am a)unsure where to go from there and b) not 100% sure on how to make a subclass of the cookiejar class?
Below I create my cookiejar object in my MainWindow constructor
view = new QWebView(this);
jar = new QNetworkCookieJar;
view->page()->networkAccessManager()->setCookieJar(jar);
And in my replyfinished slot I can see the cookie contained in the reply and I attempt to save it but nothing happens and I receive no run time errors. There isn't a great deal of stuff out there on this and have seen a few posts where the instruction was to make a subclass QNetworkCookieJar but have not made a subclass in Qt/C++ before.
Is there a simple way to store cookies, I am not looking for anything fancy. The cookies just make sure some check boxes are ticked on the login page.
// SLOT that accepts the read data from the webpage
void MainWindow::slotReplyFinished(QNetworkReply *reply){
if(reply->isFinished()){
QVariant variantCookies = reply->header(QNetworkRequest::SetCookieHeader);
QList<QNetworkCookie> cookies = qvariant_cast<QList<QNetworkCookie> >(variantCookies);
qDebug() << "Cookies reply: " << cookies;
QNetworkCookie cookie; //Create a cookie
jar = new QNetworkCookieJar;
//view->page()->networkAccessManager()->setCookieJar(jar);
jar->setCookiesFromUrl(cookies, reply->request().url());
//qDebug() << "Saved cookies: " << jar->getAllCookies();
}
qDebug() << "Network reply: " << reply->errorString() << reply->error() << reply->request().url();
}
You will need to subclass QNetworkCookieJar and in that class you should implement your own persistent storage.
class MyNetworkCookieJar : public QNetworkCookieJar {
public:
bool saveCookiesToDisk() {
// .. my implementation
return true; // if i did
}
bool loadCookiesFromDisk() {
// .. load from disk
return false; // if unable to.
}
}
The sample application from Qt project implements a persistent cookie store, it could be a good starting point for you: http://qt.gitorious.org/qt/qt/trees/4.8/demos/browser
look at cookiejar.h and cookiejar.cpp
Base of qt example, http://qt.gitorious.org/qt/qt/trees/4.8/demos/browser, i wrote this class that save and use one cookie for me. Perhaps it helps you too. Note that it just save one cookie and not list of cookies.
#include "cookiejar.h"
CookieJar::CookieJar(QObject *parent)
: QNetworkCookieJar(parent)
, m_loaded(false)
{
}
void CookieJar::load()
{
if (m_loaded)
return;
QSettings settings;
settings.beginGroup(QLatin1String("cookies"));
QList<QNetworkCookie> savedCookies = QNetworkCookie::parseCookies(settings.value("cookies").toByteArray());
for (int j = 0; j < savedCookies.count(); j++)
insertCookie(savedCookies.at(j));
m_loaded = true;
emit cookiesChanged();
}
void CookieJar::save()
{
if (!m_loaded)
return;
QList<QNetworkCookie> cookies = allCookies();
QSettings settings;
settings.beginGroup(QLatin1String("cookies"));
settings.setValue("cookies", cookies[0].toRawForm());
}
QList<QNetworkCookie> CookieJar::cookiesForUrl(const QUrl &url) const
{
// This function is called by the default QNetworkAccessManager::createRequest(),
// which adds the cookies returned by this function to the request being sent.
CookieJar *that = const_cast<CookieJar*>(this);
if (!m_loaded)
that->load();
return QNetworkCookieJar::cookiesForUrl(url);
}
bool CookieJar::setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url)
{
if (!m_loaded)
load();
QNetworkCookieJar::setCookiesFromUrl(cookieList, url);
save(); //Save cookie permanently in setting file.
emit cookiesChanged();
return true;
}
#Musa's answer is good but it only saves one cookie. I recommend using the Qt folk's implementation from the old qmlviewer located here: http://code.qt.io/cgit/qt/qt.git/tree/tools/qml/qmlruntime.cpp?h=4.7#n438
Here's the code:
class PersistentCookieJar : public QNetworkCookieJar {
public:
PersistentCookieJar(QObject *parent) : QNetworkCookieJar(parent) { load(); }
~PersistentCookieJar() { save(); }
virtual QList<QNetworkCookie> cookiesForUrl(const QUrl &url) const
{
QMutexLocker lock(&mutex);
return QNetworkCookieJar::cookiesForUrl(url);
}
virtual bool setCookiesFromUrl(const QList<QNetworkCookie> &cookieList, const QUrl &url)
{
QMutexLocker lock(&mutex);
return QNetworkCookieJar::setCookiesFromUrl(cookieList, url);
}
private:
void save()
{
QMutexLocker lock(&mutex);
QList<QNetworkCookie> list = allCookies();
QByteArray data;
foreach (QNetworkCookie cookie, list) {
if (!cookie.isSessionCookie()) {
data.append(cookie.toRawForm());
data.append("\n");
}
}
QSettings settings;
settings.setValue("Cookies",data);
}
void load()
{
QMutexLocker lock(&mutex);
QSettings settings;
QByteArray data = settings.value("Cookies").toByteArray();
setAllCookies(QNetworkCookie::parseCookies(data));
}
mutable QMutex mutex;
};