GIT_REPOSITORIES/LPP/Users/mperrinel/SciQLop-fork Commit - r1151:7dc72cc510ff

Adds "hybrid" server mode...

Alexandre Leroux -

r1151:7dc72cc510ff

parent child

Context file:

r1151:7dc72cc510ff

Collapse all files

cmake/sciqlop_applications.cmake +2 -1

              if(BUILD_TESTS)
                  INCLUDE ("cmake/sciqlop_code_coverage.cmake")
                  APPEND_COVERAGE_COMPILER_FLAGS()
              endif(BUILD_TESTS)
              #
              # Compile the diffents modules
              #
              set(sciqlop-plugin_DIR "${CMAKE_SOURCE_DIR}/plugin/cmake")
              set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${sciqlop-plugin_DIR}")
              ADD_SUBDIRECTORY("${CMAKE_SOURCE_DIR}/plugin")
              set(sciqlop-core_DIR "${CMAKE_SOURCE_DIR}/core/cmake")
              set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${sciqlop-core_DIR}")
              ADD_SUBDIRECTORY("${CMAKE_SOURCE_DIR}/core")
              set(sciqlop-gui_DIR "${CMAKE_SOURCE_DIR}/gui/cmake")
              set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${sciqlop-gui_DIR}")
              ADD_SUBDIRECTORY("${CMAKE_SOURCE_DIR}/gui")
              ADD_SUBDIRECTORY("${CMAKE_SOURCE_DIR}/app")
              OPTION (BUILD_PLUGINS "Build the plugins" OFF)
              IF(BUILD_PLUGINS)
                  set(sciqlop-mockplugin_DIR "${CMAKE_SOURCE_DIR}/plugins/mockplugin/cmake")
                  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${sciqlop-mockplugin_DIR}")
                  ADD_SUBDIRECTORY("${CMAKE_SOURCE_DIR}/plugins/mockplugin")
                  # Sets AMDA server that will be used during execution.
                  # Available values are:
                  # - "default": default AMDA server
                  # - "amdatest": AMDA test server
+                 # - "hybrid": use both the default server and the test server (the server used is relative to each product, according to its "server" property in the JSON file)
                  # Any other value will lead to the use of the default server
-                 ADD_DEFINITIONS(-DSCIQLOP_AMDA_SERVER="default")
+                 ADD_DEFINITIONS(-DSCIQLOP_AMDA_SERVER="hybrid")
                  set(sciqlop-amda_DIR "${CMAKE_SOURCE_DIR}/plugins/amda/cmake")
                  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${sciqlop-amda_DIR}")
                  ADD_SUBDIRECTORY("${CMAKE_SOURCE_DIR}/plugins/amda")
                  # Temporary target to copy to plugins dir
                  find_package(sciqlop-mockplugin)
                  find_package(sciqlop-amda)
                  ADD_CUSTOM_TARGET(plugins
                      COMMAND ${CMAKE_COMMAND} -E copy ${SCIQLOP-MOCKPLUGIN_LIBRARIES} "${LIBRARY_OUTPUT_PATH}/plugins/${SCIQLOP-MOCKPLUGIN_LIBRARIES_NAME}"
                      COMMAND ${CMAKE_COMMAND} -E copy ${SCIQLOP-AMDA_LIBRARIES} "${LIBRARY_OUTPUT_PATH}/plugins/${SCIQLOP-AMDA_LIBRARIES_NAME}"
                  )
              ENDIF(BUILD_PLUGINS)
              # LOGGER
              set(QTLOGGING_INI_FILE "${CMAKE_SOURCE_DIR}/config/QtProject/qtlogging.ini")
              FILE(COPY ${QTLOGGING_INI_FILE} DESTINATION ${CONFIG_OUTPUT_PATH})
              #
              # Code formatting
              #
              # Vera++ exclusion files
              LIST(APPEND CHECKSTYLE_EXCLUSION_FILES ${CMAKE_CURRENT_SOURCE_DIR}/formatting/vera-exclusions/exclusions.txt)
              #SCIQLOP_SET_TO_PARENT_SCOPE(CHECKSTYLE_EXCLUSION_FILES)
              INCLUDE ("cmake/sciqlop_formatting.cmake")
              #
              # Documentation generation
              #
              INCLUDE ("cmake/sciqlop_doxygen.cmake")
              #
              # Source code analysis
              #
              INCLUDE ("cmake/sciqlop_code_analysis.cmake")
              INCLUDE ("cmake/sciqlop_code_cppcheck.cmake")

meson.build +2 -1

              project('SciQLOP', 'cpp',default_options : ['cpp_std=c++14'])
              qt5 = import('qt5')
              qt5core = dependency('qt5', modules : 'Core')
              qt5widgets = dependency('qt5', modules : 'Widgets')
              qt5gui = dependency('qt5', modules : 'Gui')
              qt5svg = dependency('qt5', modules : 'Svg')
              qt5xml = dependency('qt5', modules : 'Xml')
              qt5network = dependency('qt5', modules : 'Network')
              qt5printsupport = dependency('qt5', modules : 'PrintSupport')
              qt5test = dependency('qt5', modules : 'Test')
              moc = find_program('moc-qt5','moc')
              rcc = find_program('rcc-qt5','rcc')
              if build_machine.system()=='darwin'
                add_global_link_arguments('-headerpad_max_install_names', language : 'cpp')
                install_data('build_cfg/mac/sciqlopLOGO.icns', install_dir : 'Contents/Resources')
                install_data('build_cfg/mac/Info.plist', install_dir : 'Contents')
                meson.add_install_script('build_cfg/mac/install_script.sh')
              elif host_machine.system()=='windows'
                meson.add_install_script('build_cfg/windows/install_script.sh')
              endif
              # Sets AMDA server that will be used during execution.
              # Available values are:
              # - "default": default AMDA server
              # - "amdatest": AMDA test server
+             # - "hybrid": use both the default server and the test server (the server used is relative to each product, according to its "server" property in the JSON file)
              # Any other value will lead to the use of the default server
-             add_project_arguments('-DSCIQLOP_AMDA_SERVER="default"', language : 'cpp')
+             add_project_arguments('-DSCIQLOP_AMDA_SERVER="hybrid"', language : 'cpp')
              subdir('core')
              subdir('gui')
              subdir('app')
              subdir('plugins')
              cppcheck = find_program('cppcheck', required : false)
              if cppcheck.found()
                run_target('cppcheck',
                            command : [cppcheck, '--enable=all',
                            '--project=' + join_paths(meson.build_root(), 'compile_commands.json')]
                            )
              endif

plugins/amda/include/AmdaServer.h +3 -1

              #ifndef SCIQLOP_AMDASERVER_H
              #define SCIQLOP_AMDASERVER_H
              #include <QLoggingCategory>
              #include <memory>
              Q_DECLARE_LOGGING_CATEGORY(LOG_AmdaServer)
              // Directive used to determine the active AMDA server
              #ifndef SCIQLOP_AMDA_SERVER
              #define SCIQLOP_AMDA_SERVER "default"
              #endif
              /**
               * @brief The AmdaServer class represents the server used to retrieve AMDA data (singleton).
               *
               * The server instance is initialized at compile time, as defined by the AMDA_SERVER value.
               */
              class AmdaServer {
              public:
                  /// @return the unique instance of the AMDA server
                  static AmdaServer &instance();
                  virtual ~AmdaServer() noexcept = default;
                  /// @return the name of the server
                  virtual QString name() const = 0;
+                 /// @param properties used to generate url
                  /// @return the url of the server (used to retrieve data)
-                 virtual QString url() const = 0;
+                 virtual QString url(const QVariantHash &properties = {}) const = 0;
              };
              #endif // SCIQLOP_AMDASERVER_H

plugins/amda/src/AmdaProvider.cpp +3 -2

              #include "AmdaProvider.h"
              #include "AmdaDefs.h"
              #include "AmdaResultParser.h"
              #include "AmdaServer.h"
              #include <Common/DateUtils.h>
              #include <Data/DataProviderParameters.h>
              #include <Network/NetworkController.h>
              #include <SqpApplication.h>
              #include <Variable/Variable.h>
              #include <QNetworkAccessManager>
              #include <QNetworkReply>
              #include <QTemporaryFile>
              #include <QThread>
              Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")
              namespace {
              /// URL format for a request on AMDA server. The parameters are as follows:
              /// - %1: server URL
              /// - %2: start date
              /// - %3: end date
              /// - %4: parameter id
              /// AMDA V2: http://amdatest.irap.omp.eu/php/rest/
              const auto AMDA_URL_FORMAT = QStringLiteral(
                  "http://%1/php/rest/"
                  "getParameter.php?startTime=%2&stopTime=%3&parameterID=%4&outputFormat=ASCII&"
                  "timeFormat=ISO8601&gzip=0");
              /// Dates format passed in the URL (e.g 2013-09-23T09:00)
              const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
              /// Formats a time to a date that can be passed in URL
              QString dateFormat(double sqpRange) noexcept
              {
                  auto dateTime = DateUtils::dateTime(sqpRange);
                  return dateTime.toString(AMDA_TIME_FORMAT);
              }
              AmdaResultParser::ValueType valueType(const QString &valueType)
              {
                  if (valueType == QStringLiteral("scalar")) {
                      return AmdaResultParser::ValueType::SCALAR;
                  }
                  else if (valueType == QStringLiteral("spectrogram")) {
                      return AmdaResultParser::ValueType::SPECTROGRAM;
                  }
                  else if (valueType == QStringLiteral("vector")) {
                      return AmdaResultParser::ValueType::VECTOR;
                  }
                  else {
                      return AmdaResultParser::ValueType::UNKNOWN;
                  }
              }
              } // namespace
              AmdaProvider::AmdaProvider()
              {
                  qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
                  if (auto app = sqpApp) {
                      auto &networkController = app->networkController();
                      connect(this, SIGNAL(requestConstructed(std::shared_ptr<QNetworkRequest>, QUuid,
                                                              std::function<void(QNetworkReply *, QUuid)>)),
                              &networkController,
                              SLOT(onProcessRequested(std::shared_ptr<QNetworkRequest>, QUuid,
                                                      std::function<void(QNetworkReply *, QUuid)>)));
                      connect(&sqpApp->networkController(),
                              SIGNAL(replyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)),
                              this,
                              SLOT(onReplyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)));
                  }
              }
              std::shared_ptr<IDataProvider> AmdaProvider::clone() const
              {
                  // No copy is made in the clone
                  return std::make_shared<AmdaProvider>();
              }
              void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderParameters &parameters)
              {
                  // NOTE: Try to use multithread if possible
                  const auto times = parameters.m_Times;
                  const auto data = parameters.m_Data;
                  for (const auto &dateTime : qAsConst(times)) {
                      qCDebug(LOG_AmdaProvider()) << tr("TORM AmdaProvider::requestDataLoading ") << acqIdentifier
                                                  << dateTime;
                      this->retrieveData(acqIdentifier, dateTime, data);
                      // TORM when AMDA will support quick asynchrone request
                      QThread::msleep(1000);
                  }
              }
              void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
              {
                  if (auto app = sqpApp) {
                      auto &networkController = app->networkController();
                      networkController.onReplyCanceled(acqIdentifier);
                  }
              }
              void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
                                                         std::shared_ptr<QNetworkRequest> networkRequest,
                                                         double progress)
              {
                  qCDebug(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
                                              << networkRequest.get() << progress;
                  auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
                  if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
                      // Update the progression for the current request
                      auto requestPtr = networkRequest;
                      auto findRequest = [requestPtr](const auto &entry) { return requestPtr == entry.first; };
                      auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
                      auto requestProgressMapEnd = requestProgressMap.end();
                      auto requestProgressMapIt
                          = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
                      if (requestProgressMapIt != requestProgressMapEnd) {
                          requestProgressMapIt->second = progress;
                      }
                      else {
                          // This case can happened when a progression is send after the request has been
                          // finished.
                          // Generaly the case when aborting a request
                          qCDebug(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress") << acqIdentifier
                                                      << networkRequest.get() << progress;
                      }
                      // Compute the current final progress and notify it
                      double finalProgress = 0.0;
                      auto fraq = requestProgressMap.size();
                      for (auto requestProgress : requestProgressMap) {
                          finalProgress += requestProgress.second;
                          qCDebug(LOG_AmdaProvider()) << tr("Current final progress without fraq:")
                                                      << finalProgress << requestProgress.second;
                      }
                      if (fraq > 0) {
                          finalProgress = finalProgress / fraq;
                      }
                      qCDebug(LOG_AmdaProvider()) << tr("Current final progress: ") << fraq << finalProgress;
                      emit dataProvidedProgress(acqIdentifier, finalProgress);
                  }
                  else {
                      // This case can happened when a progression is send after the request has been finished.
                      // Generaly the case when aborting a request
                      emit dataProvidedProgress(acqIdentifier, 100.0);
                  }
              }
              void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
              {
                  // Retrieves product ID from data: if the value is invalid, no request is made
                  auto productId = data.value(AMDA_XML_ID_KEY).toString();
                  if (productId.isNull()) {
                      qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
                      return;
                  }
                  // Retrieves the data type that determines whether the expected format for the result file is
                  // scalar, vector...
                  auto productValueType = valueType(data.value(AMDA_DATA_TYPE_KEY).toString());
                  // /////////// //
                  // Creates URL //
                  // /////////// //
                  auto startDate = dateFormat(dateTime.m_TStart);
                  auto endDate = dateFormat(dateTime.m_TEnd);
-                 auto url = QUrl{
-                     QString{AMDA_URL_FORMAT}.arg(AmdaServer::instance().url(), startDate, endDate, productId)};
+                 QVariantHash urlProperties{{AMDA_SERVER_KEY, data.value(AMDA_SERVER_KEY)}};
+                 auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(AmdaServer::instance().url(urlProperties),
+                                                              startDate, endDate, productId)};
                  qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
                  auto tempFile = std::make_shared<QTemporaryFile>();
                  // LAMBDA
                  auto httpDownloadFinished = [this, dateTime, tempFile,
                                               productValueType](QNetworkReply *reply, QUuid dataId) noexcept {
                      // Don't do anything if the reply was abort
                      if (reply->error() == QNetworkReply::NoError) {
                          if (tempFile) {
                              auto replyReadAll = reply->readAll();
                              if (!replyReadAll.isEmpty()) {
                                  tempFile->write(replyReadAll);
                              }
                              tempFile->close();
                              // Parse results file
                              if (auto dataSeries
                                  = AmdaResultParser::readTxt(tempFile->fileName(), productValueType)) {
                                  emit dataProvided(dataId, dataSeries, dateTime);
                              }
                              else {
                                  /// @todo ALX : debug
                                  emit dataProvidedFailed(dataId);
                              }
                          }
                          m_AcqIdToRequestProgressMap.erase(dataId);
                      }
                      else {
                          qCCritical(LOG_AmdaProvider()) << tr("httpDownloadFinished ERROR");
                          emit dataProvidedFailed(dataId);
                      }
                  };
                  auto httpFinishedLambda
                      = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
                            // Don't do anything if the reply was abort
                            if (reply->error() == QNetworkReply::NoError) {
                                auto downloadFileUrl = QUrl{QString{reply->readAll()}.trimmed()};
                                qCInfo(LOG_AmdaProvider())
                                    << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
                                // Executes request for downloading file //
                                // Creates destination file
                                if (tempFile->open()) {
                                    // Executes request and store the request for progression
                                    auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
                                    updateRequestProgress(dataId, request, 0.0);
                                    emit requestConstructed(request, dataId, httpDownloadFinished);
                                }
                                else {
                                    emit dataProvidedFailed(dataId);
                                }
                            }
                            else {
                                qCCritical(LOG_AmdaProvider()) << tr("httpFinishedLambda ERROR");
                                m_AcqIdToRequestProgressMap.erase(dataId);
                                emit dataProvidedFailed(dataId);
                            }
                        };
                  // //////////////// //
                  // Executes request //
                  // //////////////// //
                  auto request = std::make_shared<QNetworkRequest>(url);
                  qCDebug(LOG_AmdaProvider()) << tr("First Request creation") << request.get();
                  updateRequestProgress(token, request, 0.0);
                  emit requestConstructed(request, token, httpFinishedLambda);
              }
              void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
                                                       std::shared_ptr<QNetworkRequest> request, double progress)
              {
                  qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress request") << request.get();
                  auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
                  if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
                      auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
                      auto requestProgressMapIt = requestProgressMap.find(request);
                      if (requestProgressMapIt != requestProgressMap.end()) {
                          requestProgressMapIt->second = progress;
                          qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new progress for request")
                                                      << acqIdentifier << request.get() << progress;
                      }
                      else {
                          qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new request") << acqIdentifier
                                                      << request.get() << progress;
                          acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
                      }
                  }
                  else {
                      qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new acqIdentifier")
                                                  << acqIdentifier << request.get() << progress;
                      auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
                      requestProgressMap.insert(std::make_pair(request, progress));
                      m_AcqIdToRequestProgressMap.insert(
                          std::make_pair(acqIdentifier, std::move(requestProgressMap)));
                  }
              }

plugins/amda/src/AmdaServer.cpp +35 -2

              #include "AmdaServer.h"
+             #include "AmdaDefs.h"
              Q_LOGGING_CATEGORY(LOG_AmdaServer, "AmdaServer")
              namespace {
+             /// URL of the default AMDA server
+             const auto AMDA_DEFAULT_SERVER_URL = QStringLiteral("amda.irap.omp.eu");
+             /// URL of the AMDA test server
+             const auto AMDA_TEST_SERVER_URL = QStringLiteral("amdatest.irap.omp.eu");
              /// Default AMDA server
              struct AmdaDefaultServer : public AmdaServer {
              public:
                  QString name() const override { return QStringLiteral("AMDA (default)"); }
-                 QString url() const override { return QStringLiteral("amda.irap.omp.eu"); }
+                 QString url(const QVariantHash &properties) const override
+                 {
+                     Q_UNUSED(properties);
+                     return AMDA_DEFAULT_SERVER_URL;
+                 }
              };
              /// Alternative AMDA server (tests)
              struct AmdaTestServer : public AmdaServer {
              public:
                  QString name() const override { return QStringLiteral("AMDA (test)"); }
-                 QString url() const override { return QStringLiteral("amdatest.irap.omp.eu"); }
+                 QString url(const QVariantHash &properties) const override
+                 {
+                     Q_UNUSED(properties);
+                     return AMDA_TEST_SERVER_URL;
+                 }
+             };
+             /// Hybrid AMDA server: use both of default and test server.
+             /// The server used is relative to each product for which to retrieve url, according to its "server"
+             /// property
+             struct AmdaHybridServer : public AmdaServer {
+             public:
+                 QString name() const override { return QStringLiteral("AMDA (hybrid)"); }
+                 QString url(const QVariantHash &properties) const override
+                 {
+                     // Reads "server" property to determine which server url to use
+                     auto server = properties.value(AMDA_SERVER_KEY).toString();
+                     return server == QString{"amdatest"} ? AMDA_TEST_SERVER_URL : AMDA_DEFAULT_SERVER_URL;
+                 }
              };
              /// @return an AMDA server instance created from the name of the server passed in parameter. If the
              /// name does not match any known server, a default server instance is created
              std::unique_ptr<AmdaServer> createInstance(const QString &server)
              {
                  if (server == QString{"amdatest"}) {
                      return std::make_unique<AmdaTestServer>();
                  }
+                 else if (server == QString{"hybrid"}) {
+                     return std::make_unique<AmdaHybridServer>();
+                 }
                  else {
                      if (server != QString{"default"}) {
                          qCWarning(LOG_AmdaServer())
                              << QObject::tr("Unknown server '%1': default AMDA server will be used").arg(server);
                      }
                      return std::make_unique<AmdaDefaultServer>();
                  }
              }
              } // namespace
              AmdaServer &AmdaServer::instance()
              {
                  // Creates instance depending on the SCIQLOP_AMDA_SERVER value at compile time
                  static auto instance = createInstance(SCIQLOP_AMDA_SERVER);
                  return *instance;
              }

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages