GIT_REPOSITORIES/LPP/Users/mperrinel/SCIQLOP-Initialisation Commit - r693:f189aafd213c

Implementation of progression

perrinel -

r693:f189aafd213c

parent child

Context file:

r693:f189aafd213c

core/include/Data/AcquisitionRequest.h +2 0

                 {
                     m_AcqIdentifier = QUuid::createUuid();
                     m_Size = 0;
+                    m_Progression = 0;
                 }
                 QUuid m_VarRequestId;
                 SqpRange m_RangeRequested;
                 SqpRange m_CacheRangeRequested;
                 int m_Size;
+                int m_Progression;
                 std::shared_ptr<IDataProvider> m_Provider;
             };

core/include/Network/NetworkController.h +2 -1

             signals:
                 void replyFinished(QNetworkReply *reply, QUuid identifier);
-                void replyDownloadProgress(QUuid identifier, double progress);
+                void replyDownloadProgress(QUuid identifier, const QNetworkRequest &networkRequest,
+                                           double progress);
             private:
                 void waitForFinish();

core/src/Network/NetworkController.cpp +4 -4

                 impl->m_NetworkReplyToVariableId[reply] = identifier;
                 impl->unlock();
-                auto onReplyFinished = [reply, this, identifier, callback]() {
+                auto onReplyFinished = [request, reply, this, identifier, callback]() {
                     qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyFinished")
                                                      << QThread::currentThread() << reply;
                         callback(reply, identifier);
                         reply->deleteLater();
-                        emit this->replyDownloadProgress(identifier, 0);
+                        emit this->replyDownloadProgress(identifier, request, 0);
                     }
                     qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyFinished END")
                                                      << QThread::currentThread() << reply;
                 };
-                auto onReplyProgress = [reply, this](qint64 bytesRead, qint64 totalBytes) {
+                auto onReplyProgress = [reply, request, this](qint64 bytesRead, qint64 totalBytes) {
                     double progress = (bytesRead * 100.0) / totalBytes;
                     qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyProgress") << progress
                     auto it = impl->m_NetworkReplyToVariableId.find(reply);
                     impl->unlock();
                     if (it != impl->m_NetworkReplyToVariableId.cend()) {
-                        emit this->replyDownloadProgress(it->second, progress);
+                        emit this->replyDownloadProgress(it->second, request, progress);
                     }
                     qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyProgress END")
                                                      << QThread::currentThread() << reply;

core/src/Variable/VariableAcquisitionWorker.cpp +68 -19

@@ -12,6 +12,8
12	#include <QReadWriteLock>	12	#include <QReadWriteLock>
13	#include <QThread>	13	#include <QThread>
14		14
		15	#include <cmath>
		16
15	Q_LOGGING_CATEGORY(LOG_VariableAcquisitionWorker, "VariableAcquisitionWorker")	17	Q_LOGGING_CATEGORY(LOG_VariableAcquisitionWorker, "VariableAcquisitionWorker")
16		18
17	struct VariableAcquisitionWorker::VariableAcquisitionWorkerPrivate {	19	struct VariableAcquisitionWorker::VariableAcquisitionWorkerPrivate {
@@ -106,14 +108,44 void VariableAcquisitionWorker::abortProgressRequested(QUuid vIdentifier)
106	void VariableAcquisitionWorker::onVariableRetrieveDataInProgress(QUuid acqIdentifier,	108	void VariableAcquisitionWorker::onVariableRetrieveDataInProgress(QUuid acqIdentifier,
107	double progress)	109	double progress)
108	{	110	{
109	// TODO	111	impl->lockRead();
		112	auto aIdToARit = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);
		113	if (aIdToARit != impl->m_AcqIdentifierToAcqRequestMap.cend()) {
		114	auto currentPartSize = (aIdToARit->second.m_Size != 0) ? 100 / aIdToARit->second.m_Size : 0;
		115
		116	auto currentPartProgress
		117	= std::isnan(progress) ? 0.0 : (progress * currentPartSize) / 100.0;
		118	auto currentAlreadyProgress = aIdToARit->second.m_Progression * currentPartSize;
		119
		120	qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: progress :") << progress;
		121	qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress A:")
		122	<< aIdToARit->second.m_Progression
		123	<< aIdToARit->second.m_Size;
		124	qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress B:")
		125	<< currentPartSize;
		126	qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress C:")
		127	<< currentPartProgress;
		128	qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress D:")
		129	<< currentAlreadyProgress;
		130	qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress E:")
		131	<< currentAlreadyProgress + currentPartProgress
		132	<< "\n";
		133
		134	auto finalProgression = currentAlreadyProgress + currentPartProgress;
		135	emit variableRequestInProgress(aIdToARit->second.m_vIdentifier, finalProgression);
		136
		137	if (finalProgression == 100.0) {
		138	emit variableRequestInProgress(aIdToARit->second.m_vIdentifier, 0.0);
		139	}
		140	}
		141	impl->unlock();
110	}	142	}
111		143
112	void VariableAcquisitionWorker::onVariableDataAcquired(QUuid acqIdentifier,	144	void VariableAcquisitionWorker::onVariableDataAcquired(QUuid acqIdentifier,
113	std::shared_ptr<IDataSeries> dataSeries,	145	std::shared_ptr<IDataSeries> dataSeries,
114	SqpRange dataRangeAcquired)	146	SqpRange dataRangeAcquired)
115	{	147	{
116	qC~~Debug~~(LOG_VariableAcquisitionWorker()) << tr("onVariableDataAcquired on range ")	148	qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableDataAcquired on range ")
117	<< acqIdentifier << dataRangeAcquired;	149	<< acqIdentifier << dataRangeAcquired;
118	impl->lockWrite();	150	impl->lockWrite();
119	auto aIdToARit = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);	151	auto aIdToARit = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);
@@ -137,11 +169,11 void VariableAcquisitionWorker::onVariableDataAcquired(QUuid acqIdentifier,
137		169
138	// Decrement the counter of the request	170	// Decrement the counter of the request
139	auto &acqRequest = aIdToARit->second;	171	auto &acqRequest = aIdToARit->second;
140	acqRequest.m_~~Size~~ = acqRequest.m_~~Size~~ - 1;	172	acqRequest.m_Progression = acqRequest.m_Progression + 1;
141		173
142	// if the counter is 0, we can return data then run the next request if it exists and	174	// if the counter is 0, we can return data then run the next request if it exists and
143	// removed the finished request	175	// removed the finished request
144	if (acqRequest.m_Size == 0) {	176	if (acqRequest.m_Size == acqRequest.m_Progression) {
145	// Return the data	177	// Return the data
146	aIdToADPVit = impl->m_AcqIdentifierToAcqDataPacketVectorMap.find(acqIdentifier);	178	aIdToADPVit = impl->m_AcqIdentifierToAcqDataPacketVectorMap.find(acqIdentifier);
147	if (aIdToADPVit != impl->m_AcqIdentifierToAcqDataPacketVectorMap.cend()) {	179	if (aIdToADPVit != impl->m_AcqIdentifierToAcqDataPacketVectorMap.cend()) {
@@ -184,6 +216,23 void VariableAcquisitionWorker::onVariableDataAcquired(QUuid acqIdentifier,
184	impl->unlock();	216	impl->unlock();
185	}	217	}
186		218
		219	void VariableAcquisitionWorker::onExecuteRequest(QUuid acqIdentifier)
		220	{
		221	qCDebug(LOG_VariableAcquisitionWorker()) << tr("onExecuteRequest") << QThread::currentThread();
		222	impl->lockRead();
		223	auto it = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);
		224	if (it != impl->m_AcqIdentifierToAcqRequestMap.cend()) {
		225	auto request = it->second;
		226	impl->unlock();
		227	emit variableRequestInProgress(request.m_vIdentifier, 0.1);
		228	request.m_Provider->requestDataLoading(acqIdentifier, request.m_DataProviderParameters);
		229	}
		230	else {
		231	impl->unlock();
		232	// TODO log no acqIdentifier recognized
		233	}
		234	}
		235
187	void VariableAcquisitionWorker::initialize()	236	void VariableAcquisitionWorker::initialize()
188	{	237	{
189	qCDebug(LOG_VariableAcquisitionWorker()) << tr("VariableAcquisitionWorker init")	238	qCDebug(LOG_VariableAcquisitionWorker()) << tr("VariableAcquisitionWorker init")
@@ -221,18 +270,18 void VariableAcquisitionWorker::VariableAcquisitionWorkerPrivate::removeVariable
221	unlock();	270	unlock();
222	}	271	}
223		272
224	void VariableAcquisitionWorker::onExecuteRequest(QUuid acqIdentifier)	273	//void VariableAcquisitionWorker::onExecuteRequest(QUuid acqIdentifier)
225	{	274	//{
226	qCDebug(LOG_VariableAcquisitionWorker()) << tr("onExecuteRequest") << QThread::currentThread();	275	// qCDebug(LOG_VariableAcquisitionWorker()) << tr("onExecuteRequest") << QThread::currentThread();
227	impl->lockRead();	276	// impl->lockRead();
228	auto it = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);	277	// auto it = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);
229	if (it != impl->m_AcqIdentifierToAcqRequestMap.cend()) {	278	// if (it != impl->m_AcqIdentifierToAcqRequestMap.cend()) {
230	auto request = it->second;	279	// auto request = it->second;
231	impl->unlock();	280	// impl->unlock();
232	request.m_Provider->requestDataLoading(acqIdentifier, request.m_DataProviderParameters);	281	// request.m_Provider->requestDataLoading(acqIdentifier, request.m_DataProviderParameters);
233	}	282	// }
234	else {	283	// else {
235	impl->unlock();	284	// impl->unlock();
236	// TODO log no acqIdentifier recognized	285	// // TODO log no acqIdentifier recognized
237	}	286	// }
238	}	287	//} note perrinel Thu, 21 Sep 2017 14:03:42 Comment #2223 Edit Delete `\|` Has been deleted later

core/src/Variable/VariableController.cpp +2 0

             void VariableController::onVariableRetrieveDataInProgress(QUuid identifier, double progress)
             {
+                qCInfo(LOG_VariableController()) << "TORM: ariableController::onVariableRetrieveDataInProgress"
+                                                 << QThread::currentThread()->objectName() << progress;
                 if (auto var = impl->findVariable(identifier)) {
                     impl->m_VariableModel->setDataProgress(var, progress);
                 }

core/tests/Data/TestTwoDimArrayData.cpp +2 -2

                                             << true << Container{{1., 2., 3., 4., 5.},
                                                                  {6., 7., 8., 9., 10.},
                                                                  {11., 12., 13., 14., 15.}};
-                QTest::newRow("invalidInput (invalid data size")
+                QTest::newRow("invalidInput (invalid data size") << InputData{{1., 2., 3., 4., 5., 6., 7.}, 3}
-                    << InputData{{1., 2., 3., 4., 5., 6., 7.}, 3} << false << Container{{}, {}, {}};
+                                                                 << false << Container{{}, {}, {}};
                 QTest::newRow("invalidInput (less than two components")
                     << flatten(Container{{1., 2., 3., 4., 5.}}) << false << Container{{}, {}, {}};
             }

plugins/amda/include/AmdaProvider.h +11 0

             #include <QLoggingCategory>
+            #include <map>
             Q_DECLARE_LOGGING_CATEGORY(LOG_AmdaProvider)
             class QNetworkReply;
+            class QNetworkRequest;
             /**
              * @brief The AmdaProvider class is an example of how a data provider can generate data
                 void requestDataAborting(QUuid acqIdentifier) override;
+            private slots:
+                void onReplyDownloadProgress(QUuid, const QNetworkRequest &, double progress);
             private:
                 void retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data);
+                void updateRequestProgress(QUuid acqIdentifier, std::shared_ptr<QNetworkRequest> request,
+                                           double progress);
+                std::map<QUuid, std::map<std::shared_ptr<QNetworkRequest>, double> >
+                    m_AcqIdToRequestProgressMap;
             };
             #endif // SCIQLOP_AMDAPROVIDER_H

plugins/amda/src/AmdaProvider.cpp +92 -6

@@ -29,6 +29,11 const auto AMDA_URL_FORMAT = QStringLiteral(
29	/// Dates format passed in the URL (e.g 2013-09-23T09:00)	29	/// Dates format passed in the URL (e.g 2013-09-23T09:00)
30	const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");	30	const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
31		31
		32	// struct AmdaProgression {
		33	// QUuid acqIdentifier;
		34	// std::map<QNetworkRequest, double> m_RequestId;
		35	//};
		36	note perrinel Thu, 21 Sep 2017 14:03:54 Comment #2224 Edit Delete `\|` Has been deleted later
32	/// Formats a time to a date that can be passed in URL	37	/// Formats a time to a date that can be passed in URL
33	QString dateFormat(double sqpRange) noexcept	38	QString dateFormat(double sqpRange) noexcept
34	{	39	{
@@ -63,8 +68,9 AmdaProvider::AmdaProvider()
63	std::function<void(QNetworkReply *, QUuid)>)));	68	std::function<void(QNetworkReply *, QUuid)>)));
64		69
65		70
66	connect(&sqpApp->networkController(), ~~SIGNAL~~(~~replyDownloadProgress~~(~~QUuid~~, ~~double)),~~ ~~this~~,	71	connect(&sqpApp->networkController(),
67	SIGNAL(~~dataProvide~~dProgress(QUuid, double)));	72	SIGNAL(replyDownloadProgress(QUuid, const QNetworkRequest &, double)), this,
		73	SLOT(onReplyDownloadProgress(QUuid, const QNetworkRequest &, double)));
68	}	74	}
69	}	75	}
70		76
@@ -82,6 +88,7 void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderPar
82	for (const auto &dateTime : qAsConst(times)) {	88	for (const auto &dateTime : qAsConst(times)) {
83	this->retrieveData(acqIdentifier, dateTime, data);	89	this->retrieveData(acqIdentifier, dateTime, data);
84		90
		91
85	// TORM when AMDA will support quick asynchrone request	92	// TORM when AMDA will support quick asynchrone request
86	QThread::msleep(1000);	93	QThread::msleep(1000);
87	}	94	}
@@ -95,6 +102,55 void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
95	}	102	}
96	}	103	}
97		104
		105	void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
		106	const QNetworkRequest &networkRequest, double progress)
		107	{
		108	qCCritical(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << progress;
		109	auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
		110	if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
		111
		112	qCCritical(LOG_AmdaProvider()) << tr("1 onReplyDownloadProgress") << progress;
		113	auto requestPtr = &networkRequest;
		114	auto findRequest
		115	= [requestPtr](const auto &entry) { return requestPtr == entry.first.get(); };
		116
		117	auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
		118	auto requestProgressMapEnd = requestProgressMap.end();
		119	auto requestProgressMapIt
		120	= std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
		121
		122	if (requestProgressMapIt != requestProgressMapEnd) {
		123	requestProgressMapIt->second = progress;
		124	}
		125	else {
		126	qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress");
		127	}
		128	}
		129
		130	acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier); note perrinel Thu, 21 Sep 2017 14:12:54 Comment #2225 Edit Delete `\|` ok
		131	if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
		132	qCCritical(LOG_AmdaProvider()) << tr("2 onReplyDownloadProgress") << progress;
		133	double finalProgress = 0.0;
		134
		135	auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
		136	auto fraq = requestProgressMap.size();
		137
		138	for (auto requestProgress : requestProgressMap) {
		139	finalProgress += requestProgress.second;
		140	}
		141
		142	if (fraq > 0) {
		143	finalProgress = finalProgress / fraq;
		144	}
		145
		146	qCCritical(LOG_AmdaProvider()) << tr("2 onReplyDownloadProgress") << finalProgress; note perrinel Thu, 21 Sep 2017 14:13:02 Comment #2226 Edit Delete `\|` Changed later
		147	emit dataProvidedProgress(acqIdentifier, finalProgress);
		148	}
		149	else {
		150	emit dataProvidedProgress(acqIdentifier, 0.0);
		151	}
		152	}
		153
98	void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)	154	void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
99	{	155	{
100	// Retrieves product ID from data: if the value is invalid, no request is made	156	// Retrieves product ID from data: if the value is invalid, no request is made
@@ -143,6 +199,7 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVa
143	/// @todo ALX : debug	199	/// @todo ALX : debug
144	}	200	}
145	}	201	}
		202	m_AcqIdToRequestProgressMap.erase(dataId);
146	}	203	}
147		204
148	};	205	};
@@ -153,7 +210,6 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVa
153	if (reply->error() != QNetworkReply::OperationCanceledError) {	210	if (reply->error() != QNetworkReply::OperationCanceledError) {
154	auto downloadFileUrl = QUrl{QString{reply->readAll()}};	211	auto downloadFileUrl = QUrl{QString{reply->readAll()}};
155		212
156
157	qCInfo(LOG_AmdaProvider())	213	qCInfo(LOG_AmdaProvider())
158	<< tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;	214	<< tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
159	// Executes request for downloading file //	215	// Executes request for downloading file //
@@ -161,14 +217,44 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVa
161	// Creates destination file	217	// Creates destination file
162	if (tempFile->open()) {	218	if (tempFile->open()) {
163	// Executes request	219	// Executes request
164	~~emit~~ request~~Constructed~~(~~QNetworkRequest~~{downloadFileUrl}, ~~dataId~~,	220	auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
165	httpDownloadFinished);	221	updateRequestProgress(dataId, request, 0.0);
		222	emit requestConstructed(*request.get(), dataId, httpDownloadFinished);
166	}	223	}
167	}	224	}
		225	else {
		226	m_AcqIdToRequestProgressMap.erase(dataId);
		227	}
168	};	228	};
169		229
170	// //////////////// //	230	// //////////////// //
171	// Executes request //	231	// Executes request //
172	// //////////////// //	232	// //////////////// //
173	emit requestConstructed(QNetworkRequest{url}, token, httpFinishedLambda);	233
		234	auto request = std::make_shared<QNetworkRequest>(url);
		235	updateRequestProgress(token, request, 0.0);
		236
		237	emit requestConstructed(*request.get(), token, httpFinishedLambda); note perrinel Thu, 21 Sep 2017 14:19:46 Comment #2227 Edit Delete `\|` It's not the case anymore
		238	}
		239
		240	void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
		241	std::shared_ptr<QNetworkRequest> request, double progress)
		242	{
		243	auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
		244	if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
		245	auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
		246	auto requestProgressMapIt = requestProgressMap.find(request);
		247	if (requestProgressMapIt != requestProgressMap.end()) {
		248	requestProgressMapIt->second = progress;
		249	}
		250	else {
		251	acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
		252	}
		253	}
		254	else {
		255	auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
		256	requestProgressMap.insert(std::make_pair(request, progress));
		257	m_AcqIdToRequestProgressMap.insert(
		258	std::make_pair(acqIdentifier, std::move(requestProgressMap)));
		259	}
174	}	260	}

plugins/mockplugin/src/CosinusProvider.cpp +7 -2

                             progress = currentProgress;
                             emit dataProvidedProgress(acqIdentifier, progress);
+                            qCInfo(LOG_CosinusProvider()) << "TORM: CosinusProvider::retrieveData"
+                                                          << QThread::currentThread()->objectName() << progress;
+                            // NOTE: Try to use multithread if possible
                         }
                     }
                     else {
                         }
                     }
                 }
-                emit dataProvidedProgress(acqIdentifier, 0.0);
+                if (progress != 100) {
+                    // We can close progression beacause all data has been retrieved
+                    emit dataProvidedProgress(acqIdentifier, 100);
+                }
                 return std::make_shared<ScalarSeries>(std::move(xAxisData), std::move(valuesData),
                                                       Unit{QStringLiteral("t"), true}, Unit{});
             }

General Comments 2

perrinel

|

Pull request updated. Auto status change to "Under Review"

Changed commits:
  * 2 added
  * 0 removed

Changed files:
  * A core/include/Variable/VariableAcquisitionWorker.h
  * M core/include/Data/IDataProvider.h
  * M core/include/Variable/VariableController.h
  * M core/src/Network/NetworkController.cpp
  * M core/src/Variable/VariableAcquisitionWorker.cpp
  * M core/src/Variable/VariableController.cpp
  * M plugins/amda/src/AmdaProvider.cpp

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages