QWebPage/QWebFrame find-methods resulting in empty QWebElementCollection

**raven-worx** · 27th October 2012, 01:27

Hi,

i'm struggling with this problem for hours now...i hope someone can enlighten me!

I'm fetching a HTML-page with an HTTP-Request and read it with QNetworkReply::readAll(). So far so good.
Then i want to "parse" (get some information of some elements) the HTML-page using QWebPage/QWebFrame.

And there is my problem. I tried the following, but all the time an empty QWebElementCollection is returned:

Qt Code:

Switch view

QWebView view;
view.load( url );
QWebElementCollection elements = view.page()->mainFrame()->findAllElements("div");

QWebView view;
view.load( url );
QWebElementCollection elements = view.page()->mainFrame()->findAllElements("div");

To copy to clipboard, switch view to plain text mode

Qt Code:

Switch view

QWebPage page;
page.mainFrame()->setHtml( reply.readAll() );  //setContent doesn't work either
QWebElementCollection elements = page.mainFrame()->findAllElements("div");

QWebPage page;
page.mainFrame()->setHtml( reply.readAll() );  //setContent doesn't work either
QWebElementCollection elements = page.mainFrame()->findAllElements("div");

To copy to clipboard, switch view to plain text mode

Only when i set the HTML-Code directly it works just fine as expected and i get the desired elements:

Qt Code:

Switch view

QWebPage page;
page.mainFrame()->setContent( QString("<html>...</html>") );
QWebElementCollection elements = page.mainFrame()->findAllElements("div");

QWebPage page;
page.mainFrame()->setContent( QString("<html>...</html>") );
QWebElementCollection elements = page.mainFrame()->findAllElements("div");

To copy to clipboard, switch view to plain text mode

So why doesn't it work when i set the content as QByteArray? Is there some coding issue?
Note: when i load the same QByteArray into a QWebView it displays just fine.

Hope someone can finally help me with this!

EDIT:
i found out that loadFinished(bool) signal is emited with "false" for the non working cases (with the QByteArray). Thus i subclasse QWebPage and reimplemented the supportsExtension() and extension() in hope to get some info whats going wrong but the methods don't get even called (tested with debugger).

Qt Code:

Switch view

virtual bool supportsExtension( Extension extension ) const {
    return extension == ErrorPageExtension;
}
 
virtual bool extension ( Extension extension, const ExtensionOption * option = 0, ExtensionReturn * output = 0 )
{
    if (extension != QWebPage::ErrorPageExtension)
        return false;
 
    ErrorPageExtensionOption *errorOption = (ErrorPageExtensionOption*) option;
    qDebug() << "Error loading " << qPrintable(errorOption->url.toString());
    if(errorOption->domain == QWebPage::QtNetwork)
        qDebug() << "Network error (" << errorOption->error << "): ";
    else if(errorOption->domain == QWebPage::Http)
        qDebug() << "HTTP error (" << errorOption->error << "): ";
    else if(errorOption->domain == QWebPage::WebKit)
        qDebug() << "WebKit error (" << errorOption->error << "): ";
 
    qDebug() << qPrintable(errorOption->errorString);
 
    return true;
}

virtual bool supportsExtension( Extension extension ) const {
    return extension == ErrorPageExtension;
}

virtual bool extension ( Extension extension, const ExtensionOption * option = 0, ExtensionReturn * output = 0 )
{
    if (extension != QWebPage::ErrorPageExtension)
        return false;

    ErrorPageExtensionOption *errorOption = (ErrorPageExtensionOption*) option;
    qDebug() << "Error loading " << qPrintable(errorOption->url.toString());
    if(errorOption->domain == QWebPage::QtNetwork)
        qDebug() << "Network error (" << errorOption->error << "): ";
    else if(errorOption->domain == QWebPage::Http)
        qDebug() << "HTTP error (" << errorOption->error << "): ";
    else if(errorOption->domain == QWebPage::WebKit)
        qDebug() << "WebKit error (" << errorOption->error << "): ";

    qDebug() << qPrintable(errorOption->errorString);

    return true;
}

To copy to clipboard, switch view to plain text mode

**norobro** · 27th October 2012, 03:17

Perhaps I misunderstand, but the following works here:

Qt Code:

Switch view

#include <QApplication>
#include <QtWebKit>
#include <QDebug>
 
class Test : QObject{
    Q_OBJECT
public:
    Test(){
        view = new QWebView;
        view->load(QUrl("http://news.google.us"));
        connect(view, SIGNAL(loadFinished(bool)),this,SLOT(finishedLoading(bool)));
        view->show();
    }
public slots:
    void finishedLoading(bool ok){
        QWebElementCollection elements = view->page()->mainFrame()->findAllElements("div");
        QList<QWebElement> list = elements.toList();
        qDebug() << ok << list.count();
    }
private:
    QWebView *view;
};
 
#include "main.moc"
 
int main(int argc, char *argv[])
{
    QApplication app(argc, argv);
    Test test;
    return app.exec();
}

#include <QApplication>
#include <QtWebKit>
#include <QDebug>

class Test : QObject{
    Q_OBJECT
public:
    Test(){
        view = new QWebView;
        view->load(QUrl("http://news.google.us"));
        connect(view, SIGNAL(loadFinished(bool)),this,SLOT(finishedLoading(bool)));
        view->show();
    }
public slots:
    void finishedLoading(bool ok){
        QWebElementCollection elements = view->page()->mainFrame()->findAllElements("div");
        QList<QWebElement> list = elements.toList();
        qDebug() << ok << list.count();
    }
private:
    QWebView *view;
};

#include "main.moc"

int main(int argc, char *argv[])
{
    QApplication app(argc, argv);
    Test test;
    return app.exec();
}

To copy to clipboard, switch view to plain text mode

**raven-worx** · 27th October 2012, 03:44

yes this works...thank you.

But i would prefer the solution where i can specify the request all by myself and just pass the received page data to the QWebPage/QWebFrame. It's really a mistery to me why the QByteArrays are not accepted?!
In the worst case i'll have to go with the loading of the page via QWebView directly.

**norobro** · 27th October 2012, 04:55

Originally Posted by raven-worx

But i would prefer the solution where i can specify the request all by myself and just pass the received page data to the QWebPage/QWebFrame.

Ah, gotcha. Using QWebFrame::setContent() works in my simple example:

Qt Code:

Switch view

#include <QApplication>
#include <QtWebKit>
#include <QtNetwork>
#include <QDebug>
 
class Test : QObject{
    Q_OBJECT
public:
    Test(){
        QNetworkAccessManager *manager = new QNetworkAccessManager(this);
        connect(manager, SIGNAL(finished(QNetworkReply*)),
                this, SLOT(replyFinished(QNetworkReply*)));
        QNetworkRequest request(QUrl("https://news.google.com"));
        manager->get(request);
    }
public slots:
    void replyFinished(QNetworkReply *reply){
        QWebPage page;
        page.mainFrame()->setContent(reply->readAll());
        QWebElementCollection e = page.mainFrame()->findAllElements("div");
        QList<QWebElement> list = e.toList();
        qDebug() << list.count();
    }
};
 
#include "main.moc"
 
int main(int argc, char *argv[])
{
    QApplication app(argc, argv);
    Test test;
    return app.exec();
}

#include <QApplication>
#include <QtWebKit>
#include <QtNetwork>
#include <QDebug>

class Test : QObject{
    Q_OBJECT
public:
    Test(){
        QNetworkAccessManager *manager = new QNetworkAccessManager(this);
        connect(manager, SIGNAL(finished(QNetworkReply*)),
                this, SLOT(replyFinished(QNetworkReply*)));
        QNetworkRequest request(QUrl("https://news.google.com"));
        manager->get(request);
    }
public slots:
    void replyFinished(QNetworkReply *reply){
        QWebPage page;
        page.mainFrame()->setContent(reply->readAll());
        QWebElementCollection e = page.mainFrame()->findAllElements("div");
        QList<QWebElement> list = e.toList();
        qDebug() << list.count();
    }
};

#include "main.moc"

int main(int argc, char *argv[])
{
    QApplication app(argc, argv);
    Test test;
    return app.exec();
}

To copy to clipboard, switch view to plain text mode