QNetworkReply signal downloadProgress returns bytesTotal incorrectly

[Maciej]

Hello!

I am stragling with an issue with QNetworkReply. I have created my own class to manage downloading images from the Internet. It utilises QNetworkReply/QNetworkAccessManager tandem. It works fine for most of the URLs but for some does not (the problem is rather connected with hosts). E.g. http://wordpress.birds.com/wp-conten...home/bird4.jpg. The problem is that QNetworkReply class signals finished() without actually downloading proper amount of data. It corrupts any conncection. In addition, downloadProgress signal provides bytesTotal incorrectly. Returned number of bytes is 262, whilst the image is much bigger. I am using Qt 4.7.0.

sources attached:

#ifndef NETWORKIMAGE_H_
#define NETWORKIMAGE_H_
 
#include <QUrl>
#include <QString>
#include <QStringList>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QHostInfo>
#include <QTimer>
#include <QMutex>
 
#include <opencv/highgui.h>
 
class NetworkImage : public QObject{
	Q_OBJECT
public:
	NetworkImage();
	virtual ~NetworkImage();
	bool download();
	bool setUrl(QString &);
	void setImageMaximumSize(qint64 &);
	bool isRecognisedAsValidURL();
	void setVerbose(bool = true);
	void askForIP();
	QStringList getIPList();
        void saveImage(QString &);
	bool isError();
	QString getErrorString();
	void setManager(QNetworkAccessManager *);
private:
	QUrl url;
	QNetworkAccessManager *manager;
	QByteArray byteArray;
	qint64 imageMaximumSize;
	bool verbose;
	QStringList IP;
	bool errorOccured;
	QString errorString;
	QTimer timer;
	QNetworkReply* reply;
	int progress;
	qint64 bytesRecievedPreviously;
	qint64 bytesRecieved;
	bool retrieved;
	QMutex mutex;
private slots:
	void downloadFinished();
	void setIP(QHostInfo);
	void printProgress(qint64 bytesReceived, qint64 bytesTotal);
	void timerTic();
signals:
void error(QString);
void done();
IPRetrieved();
};
 
#endif /* NETWORKIMAGE_H_ */

#ifndef NETWORKIMAGE_H_
#define NETWORKIMAGE_H_

#include <QUrl>
#include <QString>
#include <QStringList>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QHostInfo>
#include <QTimer>
#include <QMutex>

#include <opencv/highgui.h>

class NetworkImage : public QObject{
	Q_OBJECT
public:
	NetworkImage();
	virtual ~NetworkImage();
	bool download();
	bool setUrl(QString &);
	void setImageMaximumSize(qint64 &);
	bool isRecognisedAsValidURL();
	void setVerbose(bool = true);
	void askForIP();
	QStringList getIPList();
        void saveImage(QString &);
	bool isError();
	QString getErrorString();
	void setManager(QNetworkAccessManager *);
private:
	QUrl url;
	QNetworkAccessManager *manager;
	QByteArray byteArray;
	qint64 imageMaximumSize;
	bool verbose;
	QStringList IP;
	bool errorOccured;
	QString errorString;
	QTimer timer;
	QNetworkReply* reply;
	int progress;
	qint64 bytesRecievedPreviously;
	qint64 bytesRecieved;
	bool retrieved;
	QMutex mutex;
private slots:
	void downloadFinished();
	void setIP(QHostInfo);
	void printProgress(qint64 bytesReceived, qint64 bytesTotal);
	void timerTic();
signals:
void error(QString);
void done();
IPRetrieved();
};

#endif /* NETWORKIMAGE_H_ */

To copy to clipboard, switch view to plain text mode 

#include "NetworkImage.h"
#include <iostream>
#include <QFile>
#include <QImage>
#include <QSettings>
#include <QMutexLocker>
 
 
NetworkImage::NetworkImage() {
	// TODO Auto-generated constructor stub
	// Set maximum size of the image to 10MB;
	verbose = false;
	errorOccured = false;
	progress = 0;
    QSettings settings("NetworkImage.config", QSettings::IniFormat);
    int time = settings.value("timeToTimeout").toInt();
    //time = 30000;
	timer.setInterval(time);
    imageMaximumSize = settings.value("imageMaximumSize").toInt();
	imageMaximumSize = 10000000;
    bytesRecievedPreviously = 0;
	bytesRecieved = 0;
	retrieved = false;
}
 
NetworkImage::~NetworkImage() {
	// TODO Auto-generated destructor stub
}
 
bool NetworkImage::setUrl(QString & textUrl)
{
 
	url.setUrl(textUrl);
	if(verbose)
	{
		std::cout << "---------------------------------------------------" << std::endl;
		std::cout << "url : "<< url.toString().toStdString() << std::endl;
		std::cout << "host : "<< url.host().toStdString() << std::endl;
	}
 
	if(isRecognisedAsValidURL())
	{
		return true;
	}
	else
	{
		errorOccured = true;
		errorString = "Error: Invalid URL";
		return false;
	}
 
}
 
bool NetworkImage::download()
{
	if(isRecognisedAsValidURL())
	{
		reply = manager->get(QNetworkRequest(url));
		qDebug() << "encoding=" << reply->rawHeader( "Content-Encoding");
		connect(reply, SIGNAL(finished()), this, SLOT(downloadFinished()));
		connect(reply, SIGNAL(downloadProgress(qint64, qint64)),this, SLOT(printProgress(qint64, qint64)) );
		connect(&timer,SIGNAL(timeout()), this, SLOT(timerTic()));
		timer.start();
		askForIP();
		if(verbose)
		{
			std::cout << "Requesting connection..." << std::endl;
		}
		return true;
	}
	else
	{
		disconnect(reply, SIGNAL(downloadProgress(qint64, qint64)),this, SLOT(printProgress(qint64, qint64)) );
		emit error("Error: Invalid URL");
		errorOccured = true;
		errorString = "Error: Invalid URL";
 
		return false;
	}
 
}
 
extern IplImage* qImage2IplImage(QImage& qImage);
void NetworkImage::downloadFinished()
{
	QMutexLocker locker(&mutex);
	retrieved = true;
	disconnect(&timer,SIGNAL(timeout()), this, SLOT(timerTic()));
	disconnect(reply, SIGNAL(downloadProgress(qint64, qint64)),this, SLOT(printProgress(qint64, qint64)) );
	if(reply->error() == QNetworkReply::NoError)
	{
		if(verbose)
			std::cout << "Download has finished" << std::endl;
 
		byteArray = reply->read(imageMaximumSize);
 
		emit done();
		//if(!cvSaveImage("output.jpg",img)) printf("Could not save: %s\n","output.jpg");
	}
	else
	{
		errorOccured = true;
	errorString = "Error: unknown error!";
 
 
		std::cerr << errorString.toStdString()  << std::endl;
		std::cout << "---------------------------------------------------" << std::endl;
		emit error(errorString);
	}
	reply->deleteLater();
	//manager->deleteLater();
    return;
}
 
void NetworkImage::setImageMaximumSize(qint64 & x)
{
	imageMaximumSize = x;
}
 
bool NetworkImage::isRecognisedAsValidURL()
{
	if(url.path().isEmpty())
	{
		std::cerr << "Error: Invalid URL!" << std::endl;
		return false;
	}
	else if(!url.isValid())
	{
		std::cerr << "Error: Invalid URL!" << std::endl;
		return false;
	}
	else if( (url.scheme() != "http") && (url.scheme() != "https") )
	{
		std::cerr << "Error: Invalid URL!" << std::endl;
		return false;
	}
	else if(url.host().isEmpty())
	{
		std::cerr << "Error: Invalid URL!" << std::endl;
		return false;
	}
	else
		return true;
}
 
void NetworkImage::setVerbose(bool x)
{
	verbose = x;
}
 
void NetworkImage::setIP(QHostInfo x)
{
	if(x.addresses().length() > 0)
	{
		for(int i=0; i<x.addresses().length();i++)
		{
			IP.append(x.addresses().at(i).toString());
			if(verbose)
			{
				for(int i=0; i<IP.length();i++)
				{
					if(verbose)
						std::cout << "IP: " << IP.at(i).toStdString() << std::endl;
				}
			}
		}
		emit IPRetrieved();
	}
}
 
void NetworkImage::askForIP()
{
	QHostInfo::lookupHost(url.host(), this, SLOT(setIP(QHostInfo)));
}
 
QStringList NetworkImage::getIPList()
{
	return IP;
}
 
void NetworkImage::saveImage(QString & name)
{
 
	//QMutexLocker locker(&mutex);
	QImage qimageMy;
 
		if(qimageMy.loadFromData(byteArray))
		{
			if(verbose)
			{
				std::cout << "Downloaded data converted into QImage successfully!" << std::endl;
				std::cout << "---------------------------------------------------" << std::endl;
			}
			IplImage* img = qImage2IplImage(qimageMy);
 
			if(!cvSaveImage(name.toStdString().c_str(),img)) printf("Could not save\n");
			return;
 
		}
		else
		{
			errorOccured = true;
			errorString = "Error: Downloaded data does not contain proper image format";
			std::cerr << "Error: Downloaded data does not contain proper image format for QImage!" << std::endl;
			emit error("Error: Downloaded data does not contain proper image format for QImage!");
			return;
		}
 
}
 
 
bool NetworkImage::isError()
{
	return errorOccured;
}
 
 
QString NetworkImage::getErrorString()
{
	return errorString;
}
 
void NetworkImage::printProgress(qint64 bytesRec, qint64 bytesTotal)
{
	int oldProgress;
	{
		QMutexLocker locker(&mutex);
		oldProgress = progress;
		bytesRecievedPreviously = bytesRecieved;
		bytesRecieved = bytesRec;
	}
	if(bytesTotal>0)
	{
		std::cout << bytesRec <<std::endl;
		std::cout << bytesTotal <<std::endl;
		progress = (int)(bytesRec*100/bytesTotal);
		if((progress/10 >  oldProgress/10))
			std::cout<< (int)(bytesRec*100/bytesTotal) << " % " <<std::endl;
	}
}
 
void NetworkImage::timerTic()
{
	if(!retrieved)
	{
		QMutexLocker locker(&mutex);
		errorOccured = true;
		errorString = "Error: Waited to long for any progress";
		std::cerr << "Error: Waited to long for any progress!" << std::endl;
		emit error("Error: Waited to long for any progress!");
	}
}

#include "NetworkImage.h"
#include <iostream>
#include <QFile>
#include <QImage>
#include <QSettings>
#include <QMutexLocker>


NetworkImage::NetworkImage() {
	// TODO Auto-generated constructor stub
	// Set maximum size of the image to 10MB;
	verbose = false;
	errorOccured = false;
	progress = 0;
    QSettings settings("NetworkImage.config", QSettings::IniFormat);
    int time = settings.value("timeToTimeout").toInt();
    //time = 30000;
	timer.setInterval(time);
    imageMaximumSize = settings.value("imageMaximumSize").toInt();
	imageMaximumSize = 10000000;
    bytesRecievedPreviously = 0;
	bytesRecieved = 0;
	retrieved = false;
}

NetworkImage::~NetworkImage() {
	// TODO Auto-generated destructor stub
}

bool NetworkImage::setUrl(QString & textUrl)
{

	url.setUrl(textUrl);
	if(verbose)
	{
		std::cout << "---------------------------------------------------" << std::endl;
		std::cout << "url : "<< url.toString().toStdString() << std::endl;
		std::cout << "host : "<< url.host().toStdString() << std::endl;
	}

	if(isRecognisedAsValidURL())
	{
		return true;
	}
	else
	{
		errorOccured = true;
		errorString = "Error: Invalid URL";
		return false;
	}

}

bool NetworkImage::download()
{
	if(isRecognisedAsValidURL())
	{
		reply = manager->get(QNetworkRequest(url));
		qDebug() << "encoding=" << reply->rawHeader( "Content-Encoding");
		connect(reply, SIGNAL(finished()), this, SLOT(downloadFinished()));
		connect(reply, SIGNAL(downloadProgress(qint64, qint64)),this, SLOT(printProgress(qint64, qint64)) );
		connect(&timer,SIGNAL(timeout()), this, SLOT(timerTic()));
		timer.start();
		askForIP();
		if(verbose)
		{
			std::cout << "Requesting connection..." << std::endl;
		}
		return true;
	}
	else
	{
		disconnect(reply, SIGNAL(downloadProgress(qint64, qint64)),this, SLOT(printProgress(qint64, qint64)) );
		emit error("Error: Invalid URL");
		errorOccured = true;
		errorString = "Error: Invalid URL";

		return false;
	}

}

extern IplImage* qImage2IplImage(QImage& qImage);
void NetworkImage::downloadFinished()
{
	QMutexLocker locker(&mutex);
	retrieved = true;
	disconnect(&timer,SIGNAL(timeout()), this, SLOT(timerTic()));
	disconnect(reply, SIGNAL(downloadProgress(qint64, qint64)),this, SLOT(printProgress(qint64, qint64)) );
	if(reply->error() == QNetworkReply::NoError)
	{
		if(verbose)
			std::cout << "Download has finished" << std::endl;

		byteArray = reply->read(imageMaximumSize);

		emit done();
		//if(!cvSaveImage("output.jpg",img)) printf("Could not save: %s\n","output.jpg");
	}
	else
	{
		errorOccured = true;
	errorString = "Error: unknown error!";


		std::cerr << errorString.toStdString()  << std::endl;
		std::cout << "---------------------------------------------------" << std::endl;
		emit error(errorString);
	}
	reply->deleteLater();
	//manager->deleteLater();
    return;
}

void NetworkImage::setImageMaximumSize(qint64 & x)
{
	imageMaximumSize = x;
}

bool NetworkImage::isRecognisedAsValidURL()
{
	if(url.path().isEmpty())
	{
		std::cerr << "Error: Invalid URL!" << std::endl;
		return false;
	}
	else if(!url.isValid())
	{
		std::cerr << "Error: Invalid URL!" << std::endl;
		return false;
	}
	else if( (url.scheme() != "http") && (url.scheme() != "https") )
	{
		std::cerr << "Error: Invalid URL!" << std::endl;
		return false;
	}
	else if(url.host().isEmpty())
	{
		std::cerr << "Error: Invalid URL!" << std::endl;
		return false;
	}
	else
		return true;
}

void NetworkImage::setVerbose(bool x)
{
	verbose = x;
}

void NetworkImage::setIP(QHostInfo x)
{
	if(x.addresses().length() > 0)
	{
		for(int i=0; i<x.addresses().length();i++)
		{
			IP.append(x.addresses().at(i).toString());
			if(verbose)
			{
				for(int i=0; i<IP.length();i++)
				{
					if(verbose)
						std::cout << "IP: " << IP.at(i).toStdString() << std::endl;
				}
			}
		}
		emit IPRetrieved();
	}
}

void NetworkImage::askForIP()
{
	QHostInfo::lookupHost(url.host(), this, SLOT(setIP(QHostInfo)));
}

QStringList NetworkImage::getIPList()
{
	return IP;
}

void NetworkImage::saveImage(QString & name)
{

	//QMutexLocker locker(&mutex);
	QImage qimageMy;

		if(qimageMy.loadFromData(byteArray))
		{
			if(verbose)
			{
				std::cout << "Downloaded data converted into QImage successfully!" << std::endl;
				std::cout << "---------------------------------------------------" << std::endl;
			}
			IplImage* img = qImage2IplImage(qimageMy);

			if(!cvSaveImage(name.toStdString().c_str(),img)) printf("Could not save\n");
			return;

		}
		else
		{
			errorOccured = true;
			errorString = "Error: Downloaded data does not contain proper image format";
			std::cerr << "Error: Downloaded data does not contain proper image format for QImage!" << std::endl;
			emit error("Error: Downloaded data does not contain proper image format for QImage!");
			return;
		}

}


bool NetworkImage::isError()
{
	return errorOccured;
}


QString NetworkImage::getErrorString()
{
	return errorString;
}

void NetworkImage::printProgress(qint64 bytesRec, qint64 bytesTotal)
{
	int oldProgress;
	{
		QMutexLocker locker(&mutex);
		oldProgress = progress;
		bytesRecievedPreviously = bytesRecieved;
		bytesRecieved = bytesRec;
	}
	if(bytesTotal>0)
	{
		std::cout << bytesRec <<std::endl;
		std::cout << bytesTotal <<std::endl;
		progress = (int)(bytesRec*100/bytesTotal);
		if((progress/10 >  oldProgress/10))
			std::cout<< (int)(bytesRec*100/bytesTotal) << " % " <<std::endl;
	}
}

void NetworkImage::timerTic()
{
	if(!retrieved)
	{
		QMutexLocker locker(&mutex);
		errorOccured = true;
		errorString = "Error: Waited to long for any progress";
		std::cerr << "Error: Waited to long for any progress!" << std::endl;
		emit error("Error: Waited to long for any progress!");
	}
}

To copy to clipboard, switch view to plain text mode 

Thanks in advance for your help.
Maciej Napora

[Maciej]

It seems that transition from Qt 4.7.0 to 4.7.3 solved the problem for me.