GIT_REPOSITORIES/LPP/Users/mperrinel/SCIQLOP-Initialisation Commit - r703:ba0e6f7d7791

Correction for MR

perrinel -

r703:ba0e6f7d7791

parent child

Context file:

r703:ba0e6f7d7791

core/src/Network/NetworkController.cpp +12 -5

             #include "Network/NetworkController.h"
             #include <QMutex>
             #include <QNetworkAccessManager>
             #include <QNetworkReply>
             #include <QNetworkRequest>
             #include <QReadWriteLock>
             #include <QThread>
             #include <unordered_map>
             Q_LOGGING_CATEGORY(LOG_NetworkController, "NetworkController")
             struct NetworkController::NetworkControllerPrivate {
                 explicit NetworkControllerPrivate(NetworkController *parent) : m_WorkingMutex{} {}
                 void lockRead() { m_Lock.lockForRead(); }
                 void lockWrite() { m_Lock.lockForWrite(); }
                 void unlock() { m_Lock.unlock(); }
                 QMutex m_WorkingMutex;
                 QReadWriteLock m_Lock;
                 std::unordered_map<QNetworkReply *, QUuid> m_NetworkReplyToId;
                 std::unique_ptr<QNetworkAccessManager> m_AccessManager{nullptr};
             };
             NetworkController::NetworkController(QObject *parent)
                     : QObject(parent), impl{spimpl::make_unique_impl<NetworkControllerPrivate>(this)}
             {
             }
             void NetworkController::onProcessRequested(std::shared_ptr<QNetworkRequest> request,
                                                        QUuid identifier,
                                                        std::function<void(QNetworkReply *, QUuid)> callback)
             {
                 qCDebug(LOG_NetworkController()) << tr("NetworkController onProcessRequested")
                                                  << QThread::currentThread()->objectName() << &request;
                 auto reply = impl->m_AccessManager->get(*request);
                 // Store the couple reply id
                 impl->lockWrite();
                 impl->m_NetworkReplyToId[reply] = identifier;
                 qCDebug(LOG_NetworkController()) << tr("Store for reply: ") << identifier;
                 impl->unlock();
                 auto onReplyFinished = [request, reply, this, identifier, callback]() {
                     qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyFinished")
                                                      << QThread::currentThread() << request.get() << reply;
                     impl->lockRead();
                     auto it = impl->m_NetworkReplyToId.find(reply);
-                    impl->unlock();
                     if (it != impl->m_NetworkReplyToId.cend()) {
+                        qCDebug(LOG_NetworkController()) << tr("Remove for reply: ") << it->second;
+                        impl->unlock();
                         impl->lockWrite();
-                        qCDebug(LOG_NetworkController()) << tr("Remove for reply: ")
-                                                         << impl->m_NetworkReplyToId[reply];
                         impl->m_NetworkReplyToId.erase(reply);
                         impl->unlock();
                         // Deletes reply
                         callback(reply, identifier);
                         reply->deleteLater();
                     }
+                    else {
+                        impl->unlock();
+                    }
                     qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyFinished END")
                                                      << QThread::currentThread() << reply;
                 };
                 auto onReplyProgress = [reply, request, this](qint64 bytesRead, qint64 totalBytes) {
                     // NOTE: a totalbytes of 0 can happened when a request has been aborted
                     if (totalBytes > 0) {
                         double progress = (bytesRead * 100.0) / totalBytes;
                         qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyProgress") << progress
                                                          << QThread::currentThread() << request.get() << reply
                                                          << bytesRead << totalBytes;
                         impl->lockRead();
                         auto it = impl->m_NetworkReplyToId.find(reply);
-                        impl->unlock();
                         if (it != impl->m_NetworkReplyToId.cend()) {
-                            emit this->replyDownloadProgress(it->second, request, progress);
+                            auto id = it->second;
+                            impl->unlock();
+                            emit this->replyDownloadProgress(id, request, progress);
+                        }
+                        else {
+                            impl->unlock();
                         }
                         qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyProgress END")
                                                          << QThread::currentThread() << reply;
                     }
                 };
                 connect(reply, &QNetworkReply::finished, this, onReplyFinished);
                 connect(reply, &QNetworkReply::downloadProgress, this, onReplyProgress);
                 qCDebug(LOG_NetworkController()) << tr("NetworkController registered END")
                                                  << QThread::currentThread()->objectName() << reply;
             }
             void NetworkController::initialize()
             {
                 qCDebug(LOG_NetworkController()) << tr("NetworkController init") << QThread::currentThread();
                 impl->m_WorkingMutex.lock();
                 impl->m_AccessManager = std::make_unique<QNetworkAccessManager>();
                 auto onReplyErrors = [this](QNetworkReply *reply, const QList<QSslError> &errors) {
                     qCCritical(LOG_NetworkController()) << tr("NetworkAcessManager errors: ") << errors;
                 };
                 connect(impl->m_AccessManager.get(), &QNetworkAccessManager::sslErrors, this, onReplyErrors);
                 qCDebug(LOG_NetworkController()) << tr("NetworkController init END");
             }
             void NetworkController::finalize()
             {
                 impl->m_WorkingMutex.unlock();
             }
             void NetworkController::onReplyCanceled(QUuid identifier)
             {
                 auto findReply = [identifier](const auto &entry) { return identifier == entry.second; };
                 qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyCanceled")
                                                  << QThread::currentThread() << identifier;
                 impl->lockRead();
                 auto end = impl->m_NetworkReplyToId.cend();
                 auto it = std::find_if(impl->m_NetworkReplyToId.cbegin(), end, findReply);
                 impl->unlock();
                 if (it != end) {
                     qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyCanceled ABORT DONE")
                                                      << QThread::currentThread() << identifier;
                     it->first->abort();
                 }
                 qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyCanceled END")
                                                  << QThread::currentThread();
             }
             void NetworkController::waitForFinish()
             {
                 QMutexLocker locker{&impl->m_WorkingMutex};
             }

plugins/amda/src/AmdaProvider.cpp +15 -12

             #include "AmdaProvider.h"
             #include "AmdaDefs.h"
             #include "AmdaResultParser.h"
             #include <Common/DateUtils.h>
             #include <Data/DataProviderParameters.h>
             #include <Network/NetworkController.h>
             #include <SqpApplication.h>
             #include <Variable/Variable.h>
             #include <QNetworkAccessManager>
             #include <QNetworkReply>
             #include <QTemporaryFile>
             #include <QThread>
             Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")
             namespace {
             /// URL format for a request on AMDA server. The parameters are as follows:
             /// - %1: start date
             /// - %2: end date
             /// - %3: parameter id
             const auto AMDA_URL_FORMAT = QStringLiteral(
                 "http://amda.irap.omp.eu/php/rest/"
                 "getParameter.php?startTime=%1&stopTime=%2&parameterID=%3&outputFormat=ASCII&"
                 "timeFormat=ISO8601&gzip=0");
             /// Dates format passed in the URL (e.g 2013-09-23T09:00)
             const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
             /// Formats a time to a date that can be passed in URL
             QString dateFormat(double sqpRange) noexcept
             {
                 auto dateTime = DateUtils::dateTime(sqpRange);
                 return dateTime.toString(AMDA_TIME_FORMAT);
             }
             AmdaResultParser::ValueType valueType(const QString &valueType)
             {
                 if (valueType == QStringLiteral("scalar")) {
                     return AmdaResultParser::ValueType::SCALAR;
                 }
                 else if (valueType == QStringLiteral("vector")) {
                     return AmdaResultParser::ValueType::VECTOR;
                 }
                 else {
                     return AmdaResultParser::ValueType::UNKNOWN;
                 }
             }
             } // namespace
             AmdaProvider::AmdaProvider()
             {
                 qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
                 if (auto app = sqpApp) {
                     auto &networkController = app->networkController();
                     connect(this, SIGNAL(requestConstructed(std::shared_ptr<QNetworkRequest>, QUuid,
                                                             std::function<void(QNetworkReply *, QUuid)>)),
                             &networkController,
                             SLOT(onProcessRequested(std::shared_ptr<QNetworkRequest>, QUuid,
                                                     std::function<void(QNetworkReply *, QUuid)>)));
                     connect(&sqpApp->networkController(),
                             SIGNAL(replyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)),
                             this,
                             SLOT(onReplyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)));
                 }
             }
             std::shared_ptr<IDataProvider> AmdaProvider::clone() const
             {
                 // No copy is made in the clone
                 return std::make_shared<AmdaProvider>();
             }
             void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderParameters &parameters)
             {
                 // NOTE: Try to use multithread if possible
                 const auto times = parameters.m_Times;
                 const auto data = parameters.m_Data;
                 for (const auto &dateTime : qAsConst(times)) {
+                    qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::requestDataLoading ") << acqIdentifier
+                                               << dateTime;
                     this->retrieveData(acqIdentifier, dateTime, data);
                     // TORM when AMDA will support quick asynchrone request
                     QThread::msleep(1000);
                 }
             }
             void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
             {
                 if (auto app = sqpApp) {
                     auto &networkController = app->networkController();
                     networkController.onReplyCanceled(acqIdentifier);
                 }
             }
             void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
                                                        std::shared_ptr<QNetworkRequest> networkRequest,
                                                        double progress)
             {
-                qCInfo(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
+                qCDebug(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
                                             << networkRequest.get() << progress;
                 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
                 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
+                    // Update the progression for the current request
                     auto requestPtr = networkRequest;
                     auto findRequest = [requestPtr](const auto &entry) { return requestPtr == entry.first; };
                     auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
                     auto requestProgressMapEnd = requestProgressMap.end();
                     auto requestProgressMapIt
                         = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
                     if (requestProgressMapIt != requestProgressMapEnd) {
                         requestProgressMapIt->second = progress;
                     }
                     else {
                         // This case can happened when a progression is send after the request has been
                         // finished.
                         // Generaly the case when aborting a request
                         qCWarning(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress")
                                                       << acqIdentifier << networkRequest.get() << progress;
                     }
-                acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
+                    // Compute the current final progress and notify it
-                if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
                     double finalProgress = 0.0;
-                    auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
                     auto fraq = requestProgressMap.size();
                     for (auto requestProgress : requestProgressMap) {
                         finalProgress += requestProgress.second;
-                        qCDebug(LOG_AmdaProvider()) << tr("current final progress without freq:")
+                        qCDebug(LOG_AmdaProvider()) << tr("Current final progress without fraq:")
                                                     << finalProgress << requestProgress.second;
                     }
                     if (fraq > 0) {
                         finalProgress = finalProgress / fraq;
                     }
-                    qCDebug(LOG_AmdaProvider()) << tr("2 onReplyDownloadProgress final progress") << fraq
+                    qCDebug(LOG_AmdaProvider()) << tr("Current final progress: ") << fraq << finalProgress;
-                                                << finalProgress;
                     emit dataProvidedProgress(acqIdentifier, finalProgress);
                 }
                 else {
                     // This case can happened when a progression is send after the request has been finished.
                     // Generaly the case when aborting a request
                     emit dataProvidedProgress(acqIdentifier, 100.0);
                 }
             }
             void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
             {
                 // Retrieves product ID from data: if the value is invalid, no request is made
                 auto productId = data.value(AMDA_XML_ID_KEY).toString();
                 if (productId.isNull()) {
                     qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
                     return;
                 }
                 // Retrieves the data type that determines whether the expected format for the result file is
                 // scalar, vector...
                 auto productValueType = valueType(data.value(AMDA_DATA_TYPE_KEY).toString());
                 // /////////// //
                 // Creates URL //
                 // /////////// //
                 auto startDate = dateFormat(dateTime.m_TStart);
                 auto endDate = dateFormat(dateTime.m_TEnd);
                 auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(startDate, endDate, productId)};
-                qCDebug(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
+                qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
                 auto tempFile = std::make_shared<QTemporaryFile>();
                 // LAMBDA
                 auto httpDownloadFinished = [this, dateTime, tempFile,
                                              productValueType](QNetworkReply *reply, QUuid dataId) noexcept {
                     // Don't do anything if the reply was abort
-                    if (reply->error() != QNetworkReply::OperationCanceledError) {
+                    if (reply->error() == QNetworkReply::NoError) {
                         if (tempFile) {
                             auto replyReadAll = reply->readAll();
                             if (!replyReadAll.isEmpty()) {
                                 tempFile->write(replyReadAll);
                             }
                             tempFile->close();
                             // Parse results file
                             if (auto dataSeries
                                 = AmdaResultParser::readTxt(tempFile->fileName(), productValueType)) {
                                 emit dataProvided(dataId, dataSeries, dateTime);
                             }
                             else {
                                 /// @todo ALX : debug
                             }
                         }
                         qCDebug(LOG_AmdaProvider()) << tr("acquisition requests erase because of finishing")
                                                     << dataId;
                         m_AcqIdToRequestProgressMap.erase(dataId);
                     }
+                    else {
+                        qCCritical(LOG_AmdaProvider()) << tr("httpDownloadFinished ERROR");
+                    }
                 };
                 auto httpFinishedLambda
                     = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
                           // Don't do anything if the reply was abort
-                          if (reply->error() != QNetworkReply::OperationCanceledError) {
+                          if (reply->error() == QNetworkReply::NoError) {
                               auto downloadFileUrl = QUrl{QString{reply->readAll()}};
-                              qCDebug(LOG_AmdaProvider())
+                              qCInfo(LOG_AmdaProvider())
                                   << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
                               // Executes request for downloading file //
                               // Creates destination file
                               if (tempFile->open()) {
                                   // Executes request and store the request for progression
                                   auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
                                   updateRequestProgress(dataId, request, 0.0);
                                   emit requestConstructed(request, dataId, httpDownloadFinished);
                               }
                           }
                           else {
                               qCDebug(LOG_AmdaProvider())
                                   << tr("acquisition requests erase because of aborting") << dataId;
+                              qCCritical(LOG_AmdaProvider()) << tr("httpFinishedLambda ERROR");
                               m_AcqIdToRequestProgressMap.erase(dataId);
                           }
                       };
                 // //////////////// //
                 // Executes request //
                 // //////////////// //
                 auto request = std::make_shared<QNetworkRequest>(url);
                 qCDebug(LOG_AmdaProvider()) << tr("First Request creation") << request.get();
                 updateRequestProgress(token, request, 0.0);
                 emit requestConstructed(request, token, httpFinishedLambda);
             }
             void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
                                                      std::shared_ptr<QNetworkRequest> request, double progress)
             {
                 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
                 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
                     auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
                     auto requestProgressMapIt = requestProgressMap.find(request);
                     if (requestProgressMapIt != requestProgressMap.end()) {
                         requestProgressMapIt->second = progress;
                         qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new progress for request")
                                                     << acqIdentifier << request.get() << progress;
                     }
                     else {
                         qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new request") << acqIdentifier
                                                     << request.get() << progress;
                         acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
                     }
                 }
                 else {
                     qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new acqIdentifier")
                                                 << acqIdentifier << request.get() << progress;
                     auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
                     requestProgressMap.insert(std::make_pair(request, progress));
                     m_AcqIdToRequestProgressMap.insert(
                         std::make_pair(acqIdentifier, std::move(requestProgressMap)));
                 }
             }

General Comments 2

perrinel

|

Pull request updated. Auto status change to "Under Review"

Changed commits:
  * 2 added
  * 0 removed

Changed files:
  * A core/include/Variable/VariableAcquisitionWorker.h
  * M core/include/Data/IDataProvider.h
  * M core/include/Variable/VariableController.h
  * M core/src/Network/NetworkController.cpp
  * M core/src/Variable/VariableAcquisitionWorker.cpp
  * M core/src/Variable/VariableController.cpp
  * M plugins/amda/src/AmdaProvider.cpp

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages