Website crawler | find html tags ("a" "h" "meta" ...)

[terpentin86]

Hey,

thanks for the reply.

With your hints I have adapted my code to the follow and it works:

void Crawler::crawl_Page()
{
    frame = new QWebPage(this);
 
 
    QWebSettings::setObjectCacheCapacities(0,0,0);
    frame->settings()->setAttribute(QWebSettings::LocalContentCanAccessFileUrls,false);
    frame->settings()->setAttribute(QWebSettings::LocalContentCanAccessRemoteUrls,false);
 
    QObject::connect(frame->mainFrame(), SIGNAL(loadFinished(bool)),
            this, SLOT(parsingWork()));
 
 
   QFile* file = new QFile("D:/tempfile.txt");
 
   if(file->open(QIODevice::ReadOnly | QIODevice::Text))
       {
       qDebug() << "Open tempfile ";
       QString htmlContent = file->readAll();
 
       qDebug() << "Count Chars :: " << htmlContent.count();
       frame->mainFrame()->setHtml(htmlContent);
 
       doc = frame->mainFrame()->documentElement();
   }
}
 
 
void Crawler::parsingWork()
{
    qDebug() << "Start parsing content .....";
 
    QWebElementCollection linkCollection = doc.findAll("a");
    qDebug() << "Found " << linkCollection.count() << " links";
 
    foreach (QWebElement link, linkCollection)
    {
      qDebug() << "found link " << link.attribute("href");
    }
 
    qDebug() << "stop parsing content .....";
}

void Crawler::crawl_Page()
{
    frame = new QWebPage(this);


    QWebSettings::setObjectCacheCapacities(0,0,0);
    frame->settings()->setAttribute(QWebSettings::LocalContentCanAccessFileUrls,false);
    frame->settings()->setAttribute(QWebSettings::LocalContentCanAccessRemoteUrls,false);

    QObject::connect(frame->mainFrame(), SIGNAL(loadFinished(bool)),
            this, SLOT(parsingWork()));


   QFile* file = new QFile("D:/tempfile.txt");

   if(file->open(QIODevice::ReadOnly | QIODevice::Text))
       {
       qDebug() << "Open tempfile ";
       QString htmlContent = file->readAll();

       qDebug() << "Count Chars :: " << htmlContent.count();
       frame->mainFrame()->setHtml(htmlContent);

       doc = frame->mainFrame()->documentElement();
   }
}


void Crawler::parsingWork()
{
    qDebug() << "Start parsing content .....";

    QWebElementCollection linkCollection = doc.findAll("a");
    qDebug() << "Found " << linkCollection.count() << " links";

    foreach (QWebElement link, linkCollection)
    {
      qDebug() << "found link " << link.attribute("href");
    }

    qDebug() << "stop parsing content .....";
}

To copy to clipboard, switch view to plain text mode 

[anda_skoa]

I would recommend to allocate your QFile onf the stack, no need to allocate on the heap since you only need in the scope of that function.

Has the nice side effect that you don't have to delete it manually (which you currently miss).

Also, if crawl_page is called more than once, you might want to explicitly delete the QWebPage at some point, e.g. using deleteLater() in the slot connected to loadFinished()

Cheers,
_