GIT_REPOSITORIES/LPP/Users/mperrinel/SciQLop-fork Files · plugins/amda/src/AmdaProvider.cpp

Improves AMDA result parsing...

Improves AMDA result parsing Finds the unit among all the commented lines, in order to handle the case where the number of commented lines differs from one file to another

Alexandre Leroux - - Load All Authors

File last commit:

r488:5b65a4cc471b


                r492:41e897ecff03

Download file

             AmdaProvider.cpp
        
                    148 lines
            
             | 5.2 KiB
            
                | text/x-c
            
             |
                CppLexer
            
             / plugins / amda / src / AmdaProvider.cpp
          
                    History
                
                 |
                  Source
                 | Raw
                 |Copy content
                 |Copy permalink

        Alexandre Leroux
    
Inits Amda provider

              r377
            
      #include "AmdaProvider.h"

        Alexandre Leroux
    
Amda provider update (2)...

              r413
            
      #include "AmdaDefs.h"

        Alexandre Leroux
    
Amda provider (3)...

              r380
            
      #include "AmdaResultParser.h"

        Alexandre Leroux
    
Uses DateUtils

              r488
            
      #include <Common/DateUtils.h>

        Alexandre Leroux
    
Amda provider (1)...

              r378
            
      #include <Data/DataProviderParameters.h>

        perrinel
    
Modify the AmdaProvider to remove from it all network controller...

              r388
            
      #include <Network/NetworkController.h>

      #include <SqpApplication.h>

      #include <Variable/Variable.h>

        Alexandre Leroux
    
Amda provider (1)...

              r378
            
      #include <QNetworkAccessManager>

      #include <QNetworkReply>

        Alexandre Leroux
    
Amda provider (2)...

              r379
            
      #include <QTemporaryFile>

        perrinel
    
Modify the AmdaProvider to remove from it all network controller...

              r388
            
      #include <QThread>

        Alexandre Leroux
    
Inits Amda provider

              r377
            
      Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")

        Alexandre Leroux
    
Amda provider (1)...

              r378
            
      namespace {

      /// URL format for a request on AMDA server. The parameters are as follows:

      /// - %1: start date

      /// - %2: end date

      /// - %3: parameter id

      const auto AMDA_URL_FORMAT = QStringLiteral(

          "http://amda.irap.omp.eu/php/rest/"

        Alexandre Leroux
    
Removes sampling in AMDA provider

              r473
            
          "getParameter.php?startTime=%1&stopTime=%2&parameterID=%3&outputFormat=ASCII&"

        Alexandre Leroux
    
Amda provider (1)...

              r378
            
          "timeFormat=ISO8601&gzip=0");

      /// Dates format passed in the URL (e.g 2013-09-23T09:00)

        perrinel
    
Log AMDA get url of the file and the download file

              r440
            
      const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");

        Alexandre Leroux
    
Amda provider (1)...

              r378
            
      /// Formats a time to a date that can be passed in URL

      QString dateFormat(double sqpDateTime) noexcept

      {

        Alexandre Leroux
    
Uses DateUtils

              r488
            
          auto dateTime = DateUtils::dateTime(sqpDateTime);

        Alexandre Leroux
    
Amda provider (1)...

              r378
            
          return dateTime.toString(AMDA_TIME_FORMAT);

      }

      } // namespace

        Alexandre Leroux
    
Amda provider cleaning...

              r409
            
      AmdaProvider::AmdaProvider()

        Alexandre Leroux
    
Inits Amda provider

              r377
            
      {

        Alexandre Leroux
    
Minor refactoring...

              r460
            
          qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();

        perrinel
    
Modify the AmdaProvider to remove from it all network controller...

              r388
            
          if (auto app = sqpApp) {

              auto &networkController = app->networkController();

        Alexandre Leroux
    
Updates signal to be valid on Windows

              r416
            
              connect(this, SIGNAL(requestConstructed(QNetworkRequest, QUuid,

                                                      std::function<void(QNetworkReply *, QUuid)>)),

                      &networkController,

                      SLOT(onProcessRequested(QNetworkRequest, QUuid,

                                              std::function<void(QNetworkReply *, QUuid)>)));

        perrinel
    
Remove connection for progress from NC -> VC to NC -> Provider.

              r425
            
              connect(&sqpApp->networkController(), SIGNAL(replyDownloadProgress(QUuid, double)), this,

                      SIGNAL(dataProvidedProgress(QUuid, double)));

        perrinel
    
Modify the AmdaProvider to remove from it all network controller...

              r388
            
          }

        Alexandre Leroux
    
Inits Amda provider

              r377
            
      }

        Alexandre Leroux
    
Updates IDataProvider::requestDataLoading() method's signature...

              r408
            
      void AmdaProvider::requestDataLoading(QUuid token, const DataProviderParameters &parameters)

        Alexandre Leroux
    
Inits Amda provider

              r377
            
      {

          // NOTE: Try to use multithread if possible

        Alexandre Leroux
    
Updates IDataProvider::requestDataLoading() method's signature...

              r408
            
          const auto times = parameters.m_Times;

        Alexandre Leroux
    
Amda provider update (2)...

              r413
            
          const auto data = parameters.m_Data;

        Alexandre Leroux
    
Updates IDataProvider::requestDataLoading() method's signature...

              r408
            
          for (const auto &dateTime : qAsConst(times)) {

        Alexandre Leroux
    
Amda provider update (2)...

              r413
            
              retrieveData(token, dateTime, data);

        Alexandre Leroux
    
Inits Amda provider

              r377
            
          }

      }

        perrinel
    
Implement of the abort download process

              r422
            
      void AmdaProvider::requestDataAborting(QUuid identifier)

      {

          if (auto app = sqpApp) {

              auto &networkController = app->networkController();

              networkController.onReplyCanceled(identifier);

          }

      }

        Alexandre Leroux
    
Amda provider update (2)...

              r413
            
      void AmdaProvider::retrieveData(QUuid token, const SqpDateTime &dateTime, const QVariantHash &data)

        Alexandre Leroux
    
Amda provider (1)...

              r378
            
      {

        Alexandre Leroux
    
Amda provider update (2)...

              r413
            
          // Retrieves product ID from data: if the value is invalid, no request is made

          auto productId = data.value(AMDA_XML_ID_KEY).toString();

          if (productId.isNull()) {

              qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");

              return;

          }

        perrinel
    
Log AMDA get url of the file and the download file

              r440
            
          qCInfo(LOG_AmdaProvider()) << tr("AmdaProvider::retrieveData") << dateTime;

        Alexandre Leroux
    
Amda provider update (2)...

              r413
            
        Alexandre Leroux
    
Amda provider (1)...

              r378
            
          // /////////// //

          // Creates URL //

          // /////////// //

        Alexandre Leroux
    
Updates IDataProvider::requestDataLoading() method's signature...

              r408
            
          auto startDate = dateFormat(dateTime.m_TStart);

          auto endDate = dateFormat(dateTime.m_TEnd);

        Alexandre Leroux
    
Amda provider (1)...

              r378
            
          auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(startDate, endDate, productId)};

        perrinel
    
Log AMDA get url of the file and the download file

              r440
            
          qCInfo(LOG_AmdaProvider()) << tr("AmdaProvider::retrieveData url:") << url;

        perrinel
    
Modify the AmdaProvider to remove from it all network controller...

              r388
            
          auto tempFile = std::make_shared<QTemporaryFile>();

        Alexandre Leroux
    
Amda provider (1)...

              r378
            
        perrinel
    
Modify the AmdaProvider to remove from it all network controller...

              r388
            
          // LAMBDA

        Alexandre Leroux
    
Amda provider cleaning...

              r409
            
          auto httpDownloadFinished

              = [this, dateTime, tempFile, token](QNetworkReply *reply, QUuid dataId) noexcept {

                    Q_UNUSED(dataId);

        perrinel
    
Add implementation of abort impact on Amda plugin

              r431
            
                    // Don't do anything if the reply was abort

                    if (reply->error() != QNetworkReply::OperationCanceledError) {

                        if (tempFile) {

                            auto replyReadAll = reply->readAll();

                            if (!replyReadAll.isEmpty()) {

                                tempFile->write(replyReadAll);

                            }

                            tempFile->close();

                            // Parse results file

                            if (auto dataSeries = AmdaResultParser::readTxt(tempFile->fileName())) {

                                emit dataProvided(token, dataSeries, dateTime);

                            }

                            else {

                                /// @todo ALX : debug

                            }

        Alexandre Leroux
    
Amda provider cleaning...

              r409
            
                        }

                    }

                };

        perrinel
    
Add implementation of abort impact on Amda plugin

              r431
            
          auto httpFinishedLambda

              = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {

        Alexandre Leroux
    
Amda provider (2)...

              r379
            
        perrinel
    
Add implementation of abort impact on Amda plugin

              r431
            
                    // Don't do anything if the reply was abort

                    if (reply->error() != QNetworkReply::OperationCanceledError) {

                        auto downloadFileUrl = QUrl{QString{reply->readAll()}};

        perrinel
    
Implement of the abort download process

              r422
            
        Alexandre Leroux
    
Amda provider (2)...

              r379
            
        perrinel
    
Log AMDA get url of the file and the download file

              r440
            
                        qCInfo(LOG_AmdaProvider()) << tr("AmdaProvider::retrieveData downloadFileUrl:")

                                                   << downloadFileUrl;

        perrinel
    
Add implementation of abort impact on Amda plugin

              r431
            
                        // Executes request for downloading file //

        perrinel
    
Modify the AmdaProvider to remove from it all network controller...

              r388
            
        perrinel
    
Add implementation of abort impact on Amda plugin

              r431
            
                        // Creates destination file

                        if (tempFile->open()) {

                            // Executes request

                            emit requestConstructed(QNetworkRequest{downloadFileUrl}, dataId,

                                                    httpDownloadFinished);

                        }

                    }

                };

        perrinel
    
Modify the AmdaProvider to remove from it all network controller...

              r388
            
          // //////////////// //

          // Executes request //

          // //////////////// //

          emit requestConstructed(QNetworkRequest{url}, token, httpFinishedLambda);

        Alexandre Leroux
    
Inits Amda provider

              r377
            
      }

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

Alexandre Leroux Inits Amda provider	r377	#include "AmdaProvider.h"
Alexandre Leroux Amda provider update (2)...	r413	#include "AmdaDefs.h"
Alexandre Leroux Amda provider (3)...	r380	#include "AmdaResultParser.h"

Alexandre Leroux Uses DateUtils	r488	#include <Common/DateUtils.h>
Alexandre Leroux Amda provider (1)...	r378	#include <Data/DataProviderParameters.h>
perrinel Modify the AmdaProvider to remove from it all network controller...	r388	#include <Network/NetworkController.h>
		#include <SqpApplication.h>
		#include <Variable/Variable.h>
Alexandre Leroux Amda provider (1)...	r378
		#include <QNetworkAccessManager>
		#include <QNetworkReply>
Alexandre Leroux Amda provider (2)...	r379	#include <QTemporaryFile>
perrinel Modify the AmdaProvider to remove from it all network controller...	r388	#include <QThread>
Alexandre Leroux Inits Amda provider	r377
		Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")

Alexandre Leroux Amda provider (1)...	r378	namespace {

		/// URL format for a request on AMDA server. The parameters are as follows:
		/// - %1: start date
		/// - %2: end date
		/// - %3: parameter id
		const auto AMDA_URL_FORMAT = QStringLiteral(
		"http://amda.irap.omp.eu/php/rest/"
Alexandre Leroux Removes sampling in AMDA provider	r473	"getParameter.php?startTime=%1&stopTime=%2&parameterID=%3&outputFormat=ASCII&"
Alexandre Leroux Amda provider (1)...	r378	"timeFormat=ISO8601&gzip=0");

		/// Dates format passed in the URL (e.g 2013-09-23T09:00)
perrinel Log AMDA get url of the file and the download file	r440	const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
Alexandre Leroux Amda provider (1)...	r378
		/// Formats a time to a date that can be passed in URL
		QString dateFormat(double sqpDateTime) noexcept
		{
Alexandre Leroux Uses DateUtils	r488	auto dateTime = DateUtils::dateTime(sqpDateTime);
Alexandre Leroux Amda provider (1)...	r378	return dateTime.toString(AMDA_TIME_FORMAT);
		}

		} // namespace

Alexandre Leroux Amda provider cleaning...	r409	AmdaProvider::AmdaProvider()
Alexandre Leroux Inits Amda provider	r377	{
Alexandre Leroux Minor refactoring...	r460	qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
perrinel Modify the AmdaProvider to remove from it all network controller...	r388	if (auto app = sqpApp) {
		auto &networkController = app->networkController();
Alexandre Leroux Updates signal to be valid on Windows	r416	connect(this, SIGNAL(requestConstructed(QNetworkRequest, QUuid,
		std::function<void(QNetworkReply *, QUuid)>)),
		&networkController,
		SLOT(onProcessRequested(QNetworkRequest, QUuid,
		std::function<void(QNetworkReply *, QUuid)>)));
perrinel Remove connection for progress from NC -> VC to NC -> Provider.	r425

		connect(&sqpApp->networkController(), SIGNAL(replyDownloadProgress(QUuid, double)), this,
		SIGNAL(dataProvidedProgress(QUuid, double)));
perrinel Modify the AmdaProvider to remove from it all network controller...	r388	}
Alexandre Leroux Inits Amda provider	r377	}

Alexandre Leroux Updates IDataProvider::requestDataLoading() method's signature...	r408	void AmdaProvider::requestDataLoading(QUuid token, const DataProviderParameters &parameters)
Alexandre Leroux Inits Amda provider	r377	{
		// NOTE: Try to use multithread if possible
Alexandre Leroux Updates IDataProvider::requestDataLoading() method's signature...	r408	const auto times = parameters.m_Times;
Alexandre Leroux Amda provider update (2)...	r413	const auto data = parameters.m_Data;
Alexandre Leroux Updates IDataProvider::requestDataLoading() method's signature...	r408	for (const auto &dateTime : qAsConst(times)) {
Alexandre Leroux Amda provider update (2)...	r413	retrieveData(token, dateTime, data);
Alexandre Leroux Inits Amda provider	r377	}
		}

perrinel Implement of the abort download process	r422	void AmdaProvider::requestDataAborting(QUuid identifier)
		{
		if (auto app = sqpApp) {
		auto &networkController = app->networkController();
		networkController.onReplyCanceled(identifier);
		}
		}

Alexandre Leroux Amda provider update (2)...	r413	void AmdaProvider::retrieveData(QUuid token, const SqpDateTime &dateTime, const QVariantHash &data)
Alexandre Leroux Amda provider (1)...	r378	{
Alexandre Leroux Amda provider update (2)...	r413	// Retrieves product ID from data: if the value is invalid, no request is made
		auto productId = data.value(AMDA_XML_ID_KEY).toString();
		if (productId.isNull()) {
		qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
		return;
		}
perrinel Log AMDA get url of the file and the download file	r440	qCInfo(LOG_AmdaProvider()) << tr("AmdaProvider::retrieveData") << dateTime;
Alexandre Leroux Amda provider update (2)...	r413
Alexandre Leroux Amda provider (1)...	r378	// /////////// //
		// Creates URL //
		// /////////// //

Alexandre Leroux Updates IDataProvider::requestDataLoading() method's signature...	r408	auto startDate = dateFormat(dateTime.m_TStart);
		auto endDate = dateFormat(dateTime.m_TEnd);
Alexandre Leroux Amda provider (1)...	r378
		auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(startDate, endDate, productId)};
perrinel Log AMDA get url of the file and the download file	r440	qCInfo(LOG_AmdaProvider()) << tr("AmdaProvider::retrieveData url:") << url;
perrinel Modify the AmdaProvider to remove from it all network controller...	r388	auto tempFile = std::make_shared<QTemporaryFile>();
Alexandre Leroux Amda provider (1)...	r378
perrinel Modify the AmdaProvider to remove from it all network controller...	r388	// LAMBDA
Alexandre Leroux Amda provider cleaning...	r409	auto httpDownloadFinished
		= [this, dateTime, tempFile, token](QNetworkReply *reply, QUuid dataId) noexcept {
		Q_UNUSED(dataId);

perrinel Add implementation of abort impact on Amda plugin	r431	// Don't do anything if the reply was abort
		if (reply->error() != QNetworkReply::OperationCanceledError) {

		if (tempFile) {
		auto replyReadAll = reply->readAll();
		if (!replyReadAll.isEmpty()) {
		tempFile->write(replyReadAll);
		}
		tempFile->close();

		// Parse results file
		if (auto dataSeries = AmdaResultParser::readTxt(tempFile->fileName())) {
		emit dataProvided(token, dataSeries, dateTime);
		}
		else {
		/// @todo ALX : debug
		}
Alexandre Leroux Amda provider cleaning...	r409	}
		}

		};
perrinel Add implementation of abort impact on Amda plugin	r431	auto httpFinishedLambda
		= [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
Alexandre Leroux Amda provider (2)...	r379
perrinel Add implementation of abort impact on Amda plugin	r431	// Don't do anything if the reply was abort
		if (reply->error() != QNetworkReply::OperationCanceledError) {
		auto downloadFileUrl = QUrl{QString{reply->readAll()}};
perrinel Implement of the abort download process	r422
Alexandre Leroux Amda provider (2)...	r379
perrinel Log AMDA get url of the file and the download file	r440	qCInfo(LOG_AmdaProvider()) << tr("AmdaProvider::retrieveData downloadFileUrl:")
		<< downloadFileUrl;
perrinel Add implementation of abort impact on Amda plugin	r431	// Executes request for downloading file //
perrinel Modify the AmdaProvider to remove from it all network controller...	r388
perrinel Add implementation of abort impact on Amda plugin	r431	// Creates destination file
		if (tempFile->open()) {
		// Executes request
		emit requestConstructed(QNetworkRequest{downloadFileUrl}, dataId,
		httpDownloadFinished);
		}
		}
		};
perrinel Modify the AmdaProvider to remove from it all network controller...	r388
		// //////////////// //
		// Executes request //
		// //////////////// //
		emit requestConstructed(QNetworkRequest{url}, token, httpFinishedLambda);
Alexandre Leroux Inits Amda provider	r377	}