GIT_REPOSITORIES/LPP/Users/mperrinel/SCIQLOP-Initialisation Commit - r693:f189aafd213c

Implementation of progression

perrinel -

r693:f189aafd213c

parent child

Context file:

r693:f189aafd213c

core/include/Data/AcquisitionRequest.h +2 0

                  {
                      m_AcqIdentifier = QUuid::createUuid();
                      m_Size = 0;
+                     m_Progression = 0;
                  }
                  QUuid m_VarRequestId;
                  SqpRange m_RangeRequested;
                  SqpRange m_CacheRangeRequested;
                  int m_Size;
+                 int m_Progression;
                  std::shared_ptr<IDataProvider> m_Provider;
              };

core/include/Network/NetworkController.h +2 -1

              signals:
                  void replyFinished(QNetworkReply *reply, QUuid identifier);
-                 void replyDownloadProgress(QUuid identifier, double progress);
+                 void replyDownloadProgress(QUuid identifier, const QNetworkRequest &networkRequest,
+                                            double progress);
              private:
                  void waitForFinish();

core/src/Network/NetworkController.cpp +4 -4

                  impl->m_NetworkReplyToVariableId[reply] = identifier;
                  impl->unlock();
-                 auto onReplyFinished = [reply, this, identifier, callback]() {
+                 auto onReplyFinished = [request, reply, this, identifier, callback]() {
                      qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyFinished")
                                                       << QThread::currentThread() << reply;
                          callback(reply, identifier);
                          reply->deleteLater();
-                         emit this->replyDownloadProgress(identifier, 0);
+                         emit this->replyDownloadProgress(identifier, request, 0);
                      }
                      qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyFinished END")
                                                       << QThread::currentThread() << reply;
                  };
-                 auto onReplyProgress = [reply, this](qint64 bytesRead, qint64 totalBytes) {
+                 auto onReplyProgress = [reply, request, this](qint64 bytesRead, qint64 totalBytes) {
                      double progress = (bytesRead * 100.0) / totalBytes;
                      qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyProgress") << progress
                      auto it = impl->m_NetworkReplyToVariableId.find(reply);
                      impl->unlock();
                      if (it != impl->m_NetworkReplyToVariableId.cend()) {
-                         emit this->replyDownloadProgress(it->second, progress);
+                         emit this->replyDownloadProgress(it->second, request, progress);
                      }
                      qCDebug(LOG_NetworkController()) << tr("NetworkController onReplyProgress END")
                                                       << QThread::currentThread() << reply;

core/src/Variable/VariableAcquisitionWorker.cpp +68 -19

              #include <QReadWriteLock>
              #include <QThread>
+             #include <cmath>
              Q_LOGGING_CATEGORY(LOG_VariableAcquisitionWorker, "VariableAcquisitionWorker")
              struct VariableAcquisitionWorker::VariableAcquisitionWorkerPrivate {
              void VariableAcquisitionWorker::onVariableRetrieveDataInProgress(QUuid acqIdentifier,
                                                                               double progress)
              {
-                 // TODO
+                 impl->lockRead();
+                 auto aIdToARit = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);
+                 if (aIdToARit != impl->m_AcqIdentifierToAcqRequestMap.cend()) {
+                     auto currentPartSize = (aIdToARit->second.m_Size != 0) ? 100 / aIdToARit->second.m_Size : 0;
+                     auto currentPartProgress
+                         = std::isnan(progress) ? 0.0 : (progress * currentPartSize) / 100.0;
+                     auto currentAlreadyProgress = aIdToARit->second.m_Progression * currentPartSize;
+                     qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: progress :") << progress;
+                     qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress A:")
+                                                             << aIdToARit->second.m_Progression
+                                                             << aIdToARit->second.m_Size;
+                     qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress B:")
+                                                             << currentPartSize;
+                     qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress C:")
+                                                             << currentPartProgress;
+                     qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress D:")
+                                                             << currentAlreadyProgress;
+                     qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableRetrieveDataInProgress E:")
+                                                             << currentAlreadyProgress + currentPartProgress
+                                                             << "\n";
+                     auto finalProgression = currentAlreadyProgress + currentPartProgress;
+                     emit variableRequestInProgress(aIdToARit->second.m_vIdentifier, finalProgression);
+                     if (finalProgression == 100.0) {
+                         emit variableRequestInProgress(aIdToARit->second.m_vIdentifier, 0.0);
+                     }
+                 }
+                 impl->unlock();
              }
              void VariableAcquisitionWorker::onVariableDataAcquired(QUuid acqIdentifier,
                                                                     std::shared_ptr<IDataSeries> dataSeries,
                                                                     SqpRange dataRangeAcquired)
              {
-                 qCDebug(LOG_VariableAcquisitionWorker()) << tr("onVariableDataAcquired on range ")
+                 qCInfo(LOG_VariableAcquisitionWorker()) << tr("TORM: onVariableDataAcquired on range ")
-                                                          << acqIdentifier << dataRangeAcquired;
                  impl->lockWrite();
                  auto aIdToARit = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);
                      // Decrement the counter of the request
                      auto &acqRequest = aIdToARit->second;
-                     acqRequest.m_Size = acqRequest.m_Size - 1;
+                     acqRequest.m_Progression = acqRequest.m_Progression + 1;
                      // if the counter is 0, we can return data then run the next request if it exists and
                      // removed the finished request
-                     if (acqRequest.m_Size == 0) {
+                     if (acqRequest.m_Size == acqRequest.m_Progression) {
                          // Return the data
                          aIdToADPVit = impl->m_AcqIdentifierToAcqDataPacketVectorMap.find(acqIdentifier);
                          if (aIdToADPVit != impl->m_AcqIdentifierToAcqDataPacketVectorMap.cend()) {
                  impl->unlock();
              }
+             void VariableAcquisitionWorker::onExecuteRequest(QUuid acqIdentifier)
+             {
+                 qCDebug(LOG_VariableAcquisitionWorker()) << tr("onExecuteRequest") << QThread::currentThread();
+                 impl->lockRead();
+                 auto it = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);
+                 if (it != impl->m_AcqIdentifierToAcqRequestMap.cend()) {
+                     auto request = it->second;
+                     impl->unlock();
+                     emit variableRequestInProgress(request.m_vIdentifier, 0.1);
+                     request.m_Provider->requestDataLoading(acqIdentifier, request.m_DataProviderParameters);
+                 }
+                 else {
+                     impl->unlock();
+                     // TODO log no acqIdentifier recognized
+                 }
+             }
              void VariableAcquisitionWorker::initialize()
              {
                  qCDebug(LOG_VariableAcquisitionWorker()) << tr("VariableAcquisitionWorker init")
                  unlock();
              }
-             void VariableAcquisitionWorker::onExecuteRequest(QUuid acqIdentifier)
+             {
-                 qCDebug(LOG_VariableAcquisitionWorker()) << tr("onExecuteRequest") << QThread::currentThread();
-                 impl->lockRead();
-                 auto it = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);
-                 if (it != impl->m_AcqIdentifierToAcqRequestMap.cend()) {
-                     auto request = it->second;
-                     impl->unlock();
-                     request.m_Provider->requestDataLoading(acqIdentifier, request.m_DataProviderParameters);
+                 }
-                 else {
-                     impl->unlock();
-                     // TODO log no acqIdentifier recognized
+                 }
+             }
+             //void VariableAcquisitionWorker::onExecuteRequest(QUuid acqIdentifier)
+             //{
+             //    qCDebug(LOG_VariableAcquisitionWorker()) << tr("onExecuteRequest") << QThread::currentThread();
+             //    impl->lockRead();
+             //    auto it = impl->m_AcqIdentifierToAcqRequestMap.find(acqIdentifier);
+             //    if (it != impl->m_AcqIdentifierToAcqRequestMap.cend()) {
+             //        auto request = it->second;
+             //        impl->unlock();
+             //        request.m_Provider->requestDataLoading(acqIdentifier, request.m_DataProviderParameters);
+             //    }
+             //    else {
+             //        impl->unlock();
+             //        // TODO log no acqIdentifier recognized
+             //    }
+             //}
            


        

    

      

  

      
          

              

                    
                        
                        note
                    

              
          
          
              
  

  
    
  
  

  

    
        perrinel
    
  

          

          
              Thu, 21 Sep 2017 14:03:42
          


          

              
                
              

          

                    
                    

            
              
              

                
                    Comment #2223
                    
                

                      
                      
                        Edit
                      
                      
                        Delete
                      
              
            

            |
                 
                 

          
      
      
          Has been deleted later

core/src/Variable/VariableController.cpp +2 0

              void VariableController::onVariableRetrieveDataInProgress(QUuid identifier, double progress)
              {
+                 qCInfo(LOG_VariableController()) << "TORM: ariableController::onVariableRetrieveDataInProgress"
+                                                  << QThread::currentThread()->objectName() << progress;
                  if (auto var = impl->findVariable(identifier)) {
                      impl->m_VariableModel->setDataProgress(var, progress);
                  }

core/tests/Data/TestTwoDimArrayData.cpp +2 -2

                                              << true << Container{{1., 2., 3., 4., 5.},
                                                                   {6., 7., 8., 9., 10.},
                                                                   {11., 12., 13., 14., 15.}};
-                 QTest::newRow("invalidInput (invalid data size")
-                     << InputData{{1., 2., 3., 4., 5., 6., 7.}, 3} << false << Container{{}, {}, {}};
+                 QTest::newRow("invalidInput (invalid data size") << InputData{{1., 2., 3., 4., 5., 6., 7.}, 3}
+                                                                  << false << Container{{}, {}, {}};
                  QTest::newRow("invalidInput (less than two components")
                      << flatten(Container{{1., 2., 3., 4., 5.}}) << false << Container{{}, {}, {}};
              }

plugins/amda/include/AmdaProvider.h +11 0

              #include <QLoggingCategory>
+             #include <map>
              Q_DECLARE_LOGGING_CATEGORY(LOG_AmdaProvider)
              class QNetworkReply;
+             class QNetworkRequest;
              /**
               * @brief The AmdaProvider class is an example of how a data provider can generate data
                  void requestDataAborting(QUuid acqIdentifier) override;
+             private slots:
+                 void onReplyDownloadProgress(QUuid, const QNetworkRequest &, double progress);
              private:
                  void retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data);
+                 void updateRequestProgress(QUuid acqIdentifier, std::shared_ptr<QNetworkRequest> request,
+                                            double progress);
+                 std::map<QUuid, std::map<std::shared_ptr<QNetworkRequest>, double> >
+                     m_AcqIdToRequestProgressMap;
              };
              #endif // SCIQLOP_AMDAPROVIDER_H

plugins/amda/src/AmdaProvider.cpp +92 -6

              /// Dates format passed in the URL (e.g 2013-09-23T09:00)
              const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
+             // struct AmdaProgression {
+             //    QUuid acqIdentifier;
+             //    std::map<QNetworkRequest, double> m_RequestId;
+             //};
+                        note
                    

              
          
          
              
  

  
    
  
  

  

    
        perrinel
    
  

          

          
              Thu, 21 Sep 2017 14:03:54
          


          

              
                
              

          

                    
                    

            
              
              

                
                    Comment #2224
                    
                

                      
                      
                        Edit
                      
                      
                        Delete
                      
              
            

            |
                 
                 

          
      
      
          Has been deleted later
              /// Formats a time to a date that can be passed in URL
              QString dateFormat(double sqpRange) noexcept
              {
                                                      std::function<void(QNetworkReply *, QUuid)>)));
-                     connect(&sqpApp->networkController(), SIGNAL(replyDownloadProgress(QUuid, double)), this,
-                             SIGNAL(dataProvidedProgress(QUuid, double)));
+                     connect(&sqpApp->networkController(),
+                             SIGNAL(replyDownloadProgress(QUuid, const QNetworkRequest &, double)), this,
+                             SLOT(onReplyDownloadProgress(QUuid, const QNetworkRequest &, double)));
                  }
              }
                  for (const auto &dateTime : qAsConst(times)) {
                      this->retrieveData(acqIdentifier, dateTime, data);
                      // TORM when AMDA will support quick asynchrone request
                      QThread::msleep(1000);
                  }
                  }
              }
+             void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
+                                                        const QNetworkRequest &networkRequest, double progress)
+             {
+                 qCCritical(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << progress;
+                 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
+                 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
+                     qCCritical(LOG_AmdaProvider()) << tr("1 onReplyDownloadProgress") << progress;
+                     auto requestPtr = &networkRequest;
+                     auto findRequest
+                         = [requestPtr](const auto &entry) { return requestPtr == entry.first.get(); };
+                     auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
+                     auto requestProgressMapEnd = requestProgressMap.end();
+                     auto requestProgressMapIt
+                         = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
+                     if (requestProgressMapIt != requestProgressMapEnd) {
+                         requestProgressMapIt->second = progress;
+                     }
+                     else {
+                         qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress");
+                     }
+                 }
+                 acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
            


        

    

      

  

      
          

              

                    
                        
                        note
                    

              
          
          
              
  

  
    
  
  

  

    
        perrinel
    
  

          

          
              Thu, 21 Sep 2017 14:12:54
          


          

              
                
              

          

                    
                    

            
              
              

                
                    Comment #2225
                    
                

                      
                      
                        Edit
                      
                      
                        Delete
                      
              
            

            |
                 
                 

          
      
      
          ok
+                 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
+                     qCCritical(LOG_AmdaProvider()) << tr("2 onReplyDownloadProgress") << progress;
+                     double finalProgress = 0.0;
+                     auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
+                     auto fraq = requestProgressMap.size();
+                     for (auto requestProgress : requestProgressMap) {
+                         finalProgress += requestProgress.second;
+                     }
+                     if (fraq > 0) {
+                         finalProgress = finalProgress / fraq;
+                     }
+                     qCCritical(LOG_AmdaProvider()) << tr("2 onReplyDownloadProgress") << finalProgress;
            


        

    

      

  

      
          

              

                    
                        
                        note
                    

              
          
          
              
  

  
    
  
  

  

    
        perrinel
    
  

          

          
              Thu, 21 Sep 2017 14:13:02
          


          

              
                
              

          

                    
                    

            
              
              

                
                    Comment #2226
                    
                

                      
                      
                        Edit
                      
                      
                        Delete
                      
              
            

            |
                 
                 

          
      
      
          Changed later
+                     emit dataProvidedProgress(acqIdentifier, finalProgress);
+                 }
+                 else {
+                     emit dataProvidedProgress(acqIdentifier, 0.0);
+                 }
+             }
              void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
              {
                  // Retrieves product ID from data: if the value is invalid, no request is made
                                  /// @todo ALX : debug
                              }
                          }
+                         m_AcqIdToRequestProgressMap.erase(dataId);
                      }
                  };
                            if (reply->error() != QNetworkReply::OperationCanceledError) {
                                auto downloadFileUrl = QUrl{QString{reply->readAll()}};
                                qCInfo(LOG_AmdaProvider())
                                    << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
                                // Executes request for downloading file //
                                // Creates destination file
                                if (tempFile->open()) {
                                    // Executes request
-                                   emit requestConstructed(QNetworkRequest{downloadFileUrl}, dataId,
-                                                           httpDownloadFinished);
+                                   auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
+                                   updateRequestProgress(dataId, request, 0.0);
+                                   emit requestConstructed(*request.get(), dataId, httpDownloadFinished);
                                }
                            }
+                           else {
+                               m_AcqIdToRequestProgressMap.erase(dataId);
+                           }
                        };
                  // //////////////// //
                  // Executes request //
                  // //////////////// //
-                 emit requestConstructed(QNetworkRequest{url}, token, httpFinishedLambda);
+                 auto request = std::make_shared<QNetworkRequest>(url);
+                 updateRequestProgress(token, request, 0.0);
+                 emit requestConstructed(*request.get(), token, httpFinishedLambda);
            


        

    

      

  

      
          

              

                    
                        
                        note
                    

              
          
          
              
  

  
    
  
  

  

    
        perrinel
    
  

          

          
              Thu, 21 Sep 2017 14:19:46
          


          

              
                
              

          

                    
                    

            
              
              

                
                    Comment #2227
                    
                

                      
                      
                        Edit
                      
                      
                        Delete
                      
              
            

            |
                 
                 

          
      
      
          It's not the case anymore
+             }
+             void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
+                                                      std::shared_ptr<QNetworkRequest> request, double progress)
+             {
+                 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
+                 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
+                     auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
+                     auto requestProgressMapIt = requestProgressMap.find(request);
+                     if (requestProgressMapIt != requestProgressMap.end()) {
+                         requestProgressMapIt->second = progress;
+                     }
+                     else {
+                         acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
+                     }
+                 }
+                 else {
+                     auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
+                     requestProgressMap.insert(std::make_pair(request, progress));
+                     m_AcqIdToRequestProgressMap.insert(
+                         std::make_pair(acqIdentifier, std::move(requestProgressMap)));
+                 }
              }

plugins/mockplugin/src/CosinusProvider.cpp +7 -2

                              progress = currentProgress;
                              emit dataProvidedProgress(acqIdentifier, progress);
+                             qCInfo(LOG_CosinusProvider()) << "TORM: CosinusProvider::retrieveData"
+                                                           << QThread::currentThread()->objectName() << progress;
+                             // NOTE: Try to use multithread if possible
                          }
                      }
                      else {
                          }
                      }
                  }
-                 emit dataProvidedProgress(acqIdentifier, 0.0);
+                 if (progress != 100) {
+                     // We can close progression beacause all data has been retrieved
+                     emit dataProvidedProgress(acqIdentifier, 100);
+                 }
                  return std::make_shared<ScalarSeries>(std::move(xAxisData), std::move(valuesData),
                                                        Unit{QStringLiteral("t"), true}, Unit{});
              }

General Comments 2

perrinel

|

Pull request updated. Auto status change to "Under Review"

Changed commits:
  * 2 added
  * 0 removed

Changed files:
  * A core/include/Variable/VariableAcquisitionWorker.h
  * M core/include/Data/IDataProvider.h
  * M core/include/Variable/VariableController.h
  * M core/src/Network/NetworkController.cpp
  * M core/src/Variable/VariableAcquisitionWorker.cpp
  * M core/src/Variable/VariableController.cpp
  * M plugins/amda/src/AmdaProvider.cpp

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages