diff --git a/core/include/Data/AcquisitionRequest.h b/core/include/Data/AcquisitionRequest.h index d9d338a..9e2b0de 100644 --- a/core/include/Data/AcquisitionRequest.h +++ b/core/include/Data/AcquisitionRequest.h @@ -21,6 +21,7 @@ struct AcquisitionRequest { { m_AcqIdentifier = QUuid::createUuid(); m_Size = 0; + m_Progression = 0; } QUuid m_VarRequestId; @@ -30,6 +31,7 @@ struct AcquisitionRequest { SqpRange m_RangeRequested; SqpRange m_CacheRangeRequested; int m_Size; + int m_Progression; std::shared_ptr m_Provider; }; diff --git a/core/include/Network/NetworkController.h b/core/include/Network/NetworkController.h index ca19b89..12b122c 100644 --- a/core/include/Network/NetworkController.h +++ b/core/include/Network/NetworkController.h @@ -36,7 +36,8 @@ public slots: signals: void replyFinished(QNetworkReply *reply, QUuid identifier); - void replyDownloadProgress(QUuid identifier, double progress); + void replyDownloadProgress(QUuid identifier, const QNetworkRequest &networkRequest, + double progress); private: void waitForFinish(); diff --git a/core/src/Network/NetworkController.cpp b/core/src/Network/NetworkController.cpp index 1649cc5..0433ef9 100644 --- a/core/src/Network/NetworkController.cpp +++ b/core/src/Network/NetworkController.cpp @@ -42,7 +42,7 @@ void NetworkController::onProcessRequested(const QNetworkRequest &request, QUuid impl->m_NetworkReplyToVariableId[reply] = identifier; impl->unlock(); - auto onReplyFinished = [reply, this, identifier, callback]() { + auto onReplyFinished = [request, reply, this, identifier, callback]() { qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyFinished") << QThread::currentThread() << reply; @@ -57,14 +57,14 @@ void NetworkController::onProcessRequested(const QNetworkRequest &request, QUuid callback(reply, identifier); reply->deleteLater(); - emit this->replyDownloadProgress(identifier, 0); + emit this->replyDownloadProgress(identifier, request, 0); } qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyFinished END") << QThread::currentThread() << reply; }; - auto onReplyProgress = [reply, this](qint64 bytesRead, qint64 totalBytes) { + auto onReplyProgress = [reply, request, this](qint64 bytesRead, qint64 totalBytes) { double progress = (bytesRead * 100.0) / totalBytes; qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyProgress") << progress @@ -73,7 +73,7 @@ void NetworkController::onProcessRequested(const QNetworkRequest &request, QUuid auto it = impl->m_NetworkReplyToVariableId.find(reply); impl->unlock(); if (it != impl->m_NetworkReplyToVariableId.cend()) { - emit this->replyDownloadProgress(it->second, progress); + emit this->replyDownloadProgress(it->second, request, progress); } qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyProgress END") << QThread::currentThread() << reply; diff --git a/core/src/Variable/VariableAcquisitionWorker.cpp b/core/src/Variable/VariableAcquisitionWorker.cpp index 93aa9bb..6b28c4f 100644 --- a/core/src/Variable/VariableAcquisitionWorker.cpp +++ b/core/src/Variable/VariableAcquisitionWorker.cpp @@ -12,6 +12,8 @@ #include #include +#include + Q_LOGGING_CATEGORY(LOG_VariableAcquisitionWorker, "VariableAcquisitionWorker") struct VariableAcquisitionWorker::VariableAcquisitionWorkerPrivate { @@ -106,15 +108,45 @@ void VariableAcquisitionWorker::abortProgressRequested(QUuid vIdentifier) void VariableAcquisitionWorker::onVariableRetrieveDataInProgress(QUuid acqIdentifier, double progress) { - // TODO + impl->lockRead(); + auto aIdToARit = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier); + if (aIdToARit != impl->m_AcqIdentifierToAcqRequestMap.cend()) { + auto currentPartSize = (aIdToARit->second.m_Size != 0) ? 100 / aIdToARit->second.m_Size : 0; + + auto currentPartProgress + = std::isnan(progress) ? 0.0 : (progress * currentPartSize) / 100.0; + auto currentAlreadyProgress = aIdToARit->second.m_Progression * currentPartSize; + + qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: progress :") << progress; + qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress A:") + << aIdToARit->second.m_Progression + << aIdToARit->second.m_Size; + qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress B:") + << currentPartSize; + qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress C:") + << currentPartProgress; + qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress D:") + << currentAlreadyProgress; + qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress E:") + << currentAlreadyProgress + currentPartProgress + << "\n"; + + auto finalProgression = currentAlreadyProgress + currentPartProgress; + emit variableRequestInProgress(aIdToARit->second.m_vIdentifier, finalProgression); + + if (finalProgression == 100.0) { + emit variableRequestInProgress(aIdToARit->second.m_vIdentifier, 0.0); + } + } + impl->unlock(); } void VariableAcquisitionWorker::onVariableDataAcquired(QUuid acqIdentifier, std::shared_ptr dataSeries, SqpRange dataRangeAcquired) { - qCDebug(LOG_VariableAcquisitionWorker()) << tr("onVariableDataAcquired on range ") - << acqIdentifier << dataRangeAcquired; + qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableDataAcquired on range ") + << acqIdentifier << dataRangeAcquired; impl->lockWrite(); auto aIdToARit = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier); if (aIdToARit != impl->m_AcqIdentifierToAcqRequestMap.cend()) { @@ -137,11 +169,11 @@ void VariableAcquisitionWorker::onVariableDataAcquired(QUuid acqIdentifier, // Decrement the counter of the request auto &acqRequest = aIdToARit->second; - acqRequest.m_Size = acqRequest.m_Size - 1; + acqRequest.m_Progression = acqRequest.m_Progression + 1; // if the counter is 0, we can return data then run the next request if it exists and // removed the finished request - if (acqRequest.m_Size == 0) { + if (acqRequest.m_Size == acqRequest.m_Progression) { // Return the data aIdToADPVit = impl->m_AcqIdentifierToAcqDataPacketVectorMap.find(acqIdentifier); if (aIdToADPVit != impl->m_AcqIdentifierToAcqDataPacketVectorMap.cend()) { @@ -192,6 +224,7 @@ void VariableAcquisitionWorker::onExecuteRequest(QUuid acqIdentifier) if (it != impl->m_AcqIdentifierToAcqRequestMap.cend()) { auto request = it->second; impl->unlock(); + emit variableRequestInProgress(request.m_vIdentifier, 0.1); request.m_Provider->requestDataLoading(acqIdentifier, request.m_DataProviderParameters); } else { diff --git a/core/src/Variable/VariableController.cpp b/core/src/Variable/VariableController.cpp index fa8e248..77c6b9e 100644 --- a/core/src/Variable/VariableController.cpp +++ b/core/src/Variable/VariableController.cpp @@ -290,6 +290,8 @@ void VariableController::onDataProvided(QUuid vIdentifier, const SqpRange &range void VariableController::onVariableRetrieveDataInProgress(QUuid identifier, double progress) { + qCInfo(LOG_VariableController()) << "TORM: ariableController::onVariableRetrieveDataInProgress" + << QThread::currentThread()->objectName() << progress; if (auto var = impl->findVariable(identifier)) { impl->m_VariableModel->setDataProgress(var, progress); } diff --git a/core/tests/Data/TestTwoDimArrayData.cpp b/core/tests/Data/TestTwoDimArrayData.cpp index f884148..713c711 100644 --- a/core/tests/Data/TestTwoDimArrayData.cpp +++ b/core/tests/Data/TestTwoDimArrayData.cpp @@ -86,8 +86,8 @@ void TestTwoDimArrayData::testCtor_data() << true << Container{{1., 2., 3., 4., 5.}, {6., 7., 8., 9., 10.}, {11., 12., 13., 14., 15.}}; - QTest::newRow("invalidInput (invalid data size") - << InputData{{1., 2., 3., 4., 5., 6., 7.}, 3} << false << Container{{}, {}, {}}; + QTest::newRow("invalidInput (invalid data size") << InputData{{1., 2., 3., 4., 5., 6., 7.}, 3} + << false << Container{{}, {}, {}}; QTest::newRow("invalidInput (less than two components") << flatten(Container{{1., 2., 3., 4., 5.}}) << false << Container{{}, {}, {}}; } diff --git a/plugins/amda/include/AmdaProvider.h b/plugins/amda/include/AmdaProvider.h index 292d9b8..d41d62e 100644 --- a/plugins/amda/include/AmdaProvider.h +++ b/plugins/amda/include/AmdaProvider.h @@ -7,10 +7,12 @@ #include +#include Q_DECLARE_LOGGING_CATEGORY(LOG_AmdaProvider) class QNetworkReply; +class QNetworkRequest; /** * @brief The AmdaProvider class is an example of how a data provider can generate data @@ -23,8 +25,17 @@ public: void requestDataAborting(QUuid acqIdentifier) override; +private slots: + void onReplyDownloadProgress(QUuid, const QNetworkRequest &, double progress); + private: void retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data); + + void updateRequestProgress(QUuid acqIdentifier, std::shared_ptr request, + double progress); + + std::map, double> > + m_AcqIdToRequestProgressMap; }; #endif // SCIQLOP_AMDAPROVIDER_H diff --git a/plugins/amda/src/AmdaProvider.cpp b/plugins/amda/src/AmdaProvider.cpp index 8fe1f67..7524f1d 100644 --- a/plugins/amda/src/AmdaProvider.cpp +++ b/plugins/amda/src/AmdaProvider.cpp @@ -29,6 +29,11 @@ const auto AMDA_URL_FORMAT = QStringLiteral( /// Dates format passed in the URL (e.g 2013-09-23T09:00) const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss"); +// struct AmdaProgression { +// QUuid acqIdentifier; +// std::map m_RequestId; +//}; + /// Formats a time to a date that can be passed in URL QString dateFormat(double sqpRange) noexcept { @@ -63,8 +68,9 @@ AmdaProvider::AmdaProvider() std::function))); - connect(&sqpApp->networkController(), SIGNAL(replyDownloadProgress(QUuid, double)), this, - SIGNAL(dataProvidedProgress(QUuid, double))); + connect(&sqpApp->networkController(), + SIGNAL(replyDownloadProgress(QUuid, const QNetworkRequest &, double)), this, + SLOT(onReplyDownloadProgress(QUuid, const QNetworkRequest &, double))); } } @@ -76,6 +82,7 @@ void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderPar for (const auto &dateTime : qAsConst(times)) { this->retrieveData(acqIdentifier, dateTime, data); + // TORM when AMDA will support quick asynchrone request QThread::msleep(1000); } @@ -89,6 +96,55 @@ void AmdaProvider::requestDataAborting(QUuid acqIdentifier) } } +void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier, + const QNetworkRequest &networkRequest, double progress) +{ + qCCritical(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << progress; + auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier); + if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) { + + qCCritical(LOG_AmdaProvider()) << tr("1 onReplyDownloadProgress") << progress; + auto requestPtr = &networkRequest; + auto findRequest + = [requestPtr](const auto &entry) { return requestPtr == entry.first.get(); }; + + auto &requestProgressMap = acqIdToRequestProgressMapIt->second; + auto requestProgressMapEnd = requestProgressMap.end(); + auto requestProgressMapIt + = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest); + + if (requestProgressMapIt != requestProgressMapEnd) { + requestProgressMapIt->second = progress; + } + else { + qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress"); + } + } + + acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier); + if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) { + qCCritical(LOG_AmdaProvider()) << tr("2 onReplyDownloadProgress") << progress; + double finalProgress = 0.0; + + auto &requestProgressMap = acqIdToRequestProgressMapIt->second; + auto fraq = requestProgressMap.size(); + + for (auto requestProgress : requestProgressMap) { + finalProgress += requestProgress.second; + } + + if (fraq > 0) { + finalProgress = finalProgress / fraq; + } + + qCCritical(LOG_AmdaProvider()) << tr("2 onReplyDownloadProgress") << finalProgress; + emit dataProvidedProgress(acqIdentifier, finalProgress); + } + else { + emit dataProvidedProgress(acqIdentifier, 0.0); + } +} + void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data) { // Retrieves product ID from data: if the value is invalid, no request is made @@ -137,6 +193,7 @@ void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVa /// @todo ALX : debug } } + m_AcqIdToRequestProgressMap.erase(dataId); } }; @@ -147,7 +204,6 @@ void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVa if (reply->error() != QNetworkReply::OperationCanceledError) { auto downloadFileUrl = QUrl{QString{reply->readAll()}}; - qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl; // Executes request for downloading file // @@ -155,14 +211,44 @@ void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVa // Creates destination file if (tempFile->open()) { // Executes request - emit requestConstructed(QNetworkRequest{downloadFileUrl}, dataId, - httpDownloadFinished); + auto request = std::make_shared(downloadFileUrl); + updateRequestProgress(dataId, request, 0.0); + emit requestConstructed(*request.get(), dataId, httpDownloadFinished); } } + else { + m_AcqIdToRequestProgressMap.erase(dataId); + } }; // //////////////// // // Executes request // // //////////////// // - emit requestConstructed(QNetworkRequest{url}, token, httpFinishedLambda); + + auto request = std::make_shared(url); + updateRequestProgress(token, request, 0.0); + + emit requestConstructed(*request.get(), token, httpFinishedLambda); +} + +void AmdaProvider::updateRequestProgress(QUuid acqIdentifier, + std::shared_ptr request, double progress) +{ + auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier); + if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) { + auto &requestProgressMap = acqIdToRequestProgressMapIt->second; + auto requestProgressMapIt = requestProgressMap.find(request); + if (requestProgressMapIt != requestProgressMap.end()) { + requestProgressMapIt->second = progress; + } + else { + acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress)); + } + } + else { + auto requestProgressMap = std::map, double>{}; + requestProgressMap.insert(std::make_pair(request, progress)); + m_AcqIdToRequestProgressMap.insert( + std::make_pair(acqIdentifier, std::move(requestProgressMap))); + } } diff --git a/plugins/mockplugin/src/CosinusProvider.cpp b/plugins/mockplugin/src/CosinusProvider.cpp index 4781d41..cb64b5c 100644 --- a/plugins/mockplugin/src/CosinusProvider.cpp +++ b/plugins/mockplugin/src/CosinusProvider.cpp @@ -52,6 +52,9 @@ std::shared_ptr CosinusProvider::retrieveData(QUuid acqIdentifier, progress = currentProgress; emit dataProvidedProgress(acqIdentifier, progress); + qCInfo(LOG_CosinusProvider()) << "TORM: CosinusProvider::retrieveData" + << QThread::currentThread()->objectName() << progress; + // NOTE: Try to use multithread if possible } } else { @@ -62,8 +65,10 @@ std::shared_ptr CosinusProvider::retrieveData(QUuid acqIdentifier, } } } - emit dataProvidedProgress(acqIdentifier, 0.0); - + if (progress != 100) { + // We can close progression beacause all data has been retrieved + emit dataProvidedProgress(acqIdentifier, 100); + } return std::make_shared(std::move(xAxisData), std::move(valuesData), Unit{QStringLiteral("t"), true}, Unit{}); }