##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Changes url
Alexandre Leroux -
r790:b4395612c294
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,284 +1,285
1 #include "AmdaProvider.h"
1 #include "AmdaProvider.h"
2 #include "AmdaDefs.h"
2 #include "AmdaDefs.h"
3 #include "AmdaResultParser.h"
3 #include "AmdaResultParser.h"
4
4
5 #include <Common/DateUtils.h>
5 #include <Common/DateUtils.h>
6 #include <Data/DataProviderParameters.h>
6 #include <Data/DataProviderParameters.h>
7 #include <Network/NetworkController.h>
7 #include <Network/NetworkController.h>
8 #include <SqpApplication.h>
8 #include <SqpApplication.h>
9 #include <Variable/Variable.h>
9 #include <Variable/Variable.h>
10
10
11 #include <QNetworkAccessManager>
11 #include <QNetworkAccessManager>
12 #include <QNetworkReply>
12 #include <QNetworkReply>
13 #include <QTemporaryFile>
13 #include <QTemporaryFile>
14 #include <QThread>
14 #include <QThread>
15
15
16 Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")
16 Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")
17
17
18 namespace {
18 namespace {
19
19
20 /// URL format for a request on AMDA server. The parameters are as follows:
20 /// URL format for a request on AMDA server. The parameters are as follows:
21 /// - %1: start date
21 /// - %1: start date
22 /// - %2: end date
22 /// - %2: end date
23 /// - %3: parameter id
23 /// - %3: parameter id
24 /// Old url: http://amda.irap.omp.eu/php/rest/
24 /// AMDA V2: http://amdatest.irap.omp.eu/php/rest/
25 const auto AMDA_URL_FORMAT = QStringLiteral(
25 const auto AMDA_URL_FORMAT = QStringLiteral(
26 "http://amdatest.irap.omp.eu/php/rest/"
26 "http://amda.irap.omp.eu/php/rest/"
27 "getParameter.php?startTime=%1&stopTime=%2&parameterID=%3&outputFormat=ASCII&"
27 "getParameter.php?startTime=%1&stopTime=%2&parameterID=%3&outputFormat=ASCII&"
28 "timeFormat=ISO8601&gzip=0");
28 "timeFormat=ISO8601&gzip=0");
29
29
30 /// Dates format passed in the URL (e.g 2013-09-23T09:00)
30 /// Dates format passed in the URL (e.g 2013-09-23T09:00)
31 const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
31 const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
32
32
33 /// Formats a time to a date that can be passed in URL
33 /// Formats a time to a date that can be passed in URL
34 QString dateFormat(double sqpRange) noexcept
34 QString dateFormat(double sqpRange) noexcept
35 {
35 {
36 auto dateTime = DateUtils::dateTime(sqpRange);
36 auto dateTime = DateUtils::dateTime(sqpRange);
37 return dateTime.toString(AMDA_TIME_FORMAT);
37 return dateTime.toString(AMDA_TIME_FORMAT);
38 }
38 }
39
39
40 AmdaResultParser::ValueType valueType(const QString &valueType)
40 AmdaResultParser::ValueType valueType(const QString &valueType)
41 {
41 {
42 if (valueType == QStringLiteral("scalar")) {
42 if (valueType == QStringLiteral("scalar")) {
43 return AmdaResultParser::ValueType::SCALAR;
43 return AmdaResultParser::ValueType::SCALAR;
44 }
44 }
45 else if (valueType == QStringLiteral("vector")) {
45 else if (valueType == QStringLiteral("vector")) {
46 return AmdaResultParser::ValueType::VECTOR;
46 return AmdaResultParser::ValueType::VECTOR;
47 }
47 }
48 else {
48 else {
49 return AmdaResultParser::ValueType::UNKNOWN;
49 return AmdaResultParser::ValueType::UNKNOWN;
50 }
50 }
51 }
51 }
52
52
53 } // namespace
53 } // namespace
54
54
55 AmdaProvider::AmdaProvider()
55 AmdaProvider::AmdaProvider()
56 {
56 {
57 qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
57 qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
58 if (auto app = sqpApp) {
58 if (auto app = sqpApp) {
59 auto &networkController = app->networkController();
59 auto &networkController = app->networkController();
60 connect(this, SIGNAL(requestConstructed(std::shared_ptr<QNetworkRequest>, QUuid,
60 connect(this, SIGNAL(requestConstructed(std::shared_ptr<QNetworkRequest>, QUuid,
61 std::function<void(QNetworkReply *, QUuid)>)),
61 std::function<void(QNetworkReply *, QUuid)>)),
62 &networkController,
62 &networkController,
63 SLOT(onProcessRequested(std::shared_ptr<QNetworkRequest>, QUuid,
63 SLOT(onProcessRequested(std::shared_ptr<QNetworkRequest>, QUuid,
64 std::function<void(QNetworkReply *, QUuid)>)));
64 std::function<void(QNetworkReply *, QUuid)>)));
65
65
66
66
67 connect(&sqpApp->networkController(),
67 connect(&sqpApp->networkController(),
68 SIGNAL(replyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)),
68 SIGNAL(replyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)),
69 this,
69 this,
70 SLOT(onReplyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)));
70 SLOT(onReplyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)));
71 }
71 }
72 }
72 }
73
73
74 std::shared_ptr<IDataProvider> AmdaProvider::clone() const
74 std::shared_ptr<IDataProvider> AmdaProvider::clone() const
75 {
75 {
76 // No copy is made in the clone
76 // No copy is made in the clone
77 return std::make_shared<AmdaProvider>();
77 return std::make_shared<AmdaProvider>();
78 }
78 }
79
79
80 void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderParameters &parameters)
80 void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderParameters &parameters)
81 {
81 {
82 // NOTE: Try to use multithread if possible
82 // NOTE: Try to use multithread if possible
83 const auto times = parameters.m_Times;
83 const auto times = parameters.m_Times;
84 const auto data = parameters.m_Data;
84 const auto data = parameters.m_Data;
85 for (const auto &dateTime : qAsConst(times)) {
85 for (const auto &dateTime : qAsConst(times)) {
86 qCDebug(LOG_AmdaProvider()) << tr("TORM AmdaProvider::requestDataLoading ") << acqIdentifier
86 qCDebug(LOG_AmdaProvider()) << tr("TORM AmdaProvider::requestDataLoading ") << acqIdentifier
87 << dateTime;
87 << dateTime;
88 this->retrieveData(acqIdentifier, dateTime, data);
88 this->retrieveData(acqIdentifier, dateTime, data);
89
89
90
90
91 // TORM when AMDA will support quick asynchrone request
91 // TORM when AMDA will support quick asynchrone request
92 QThread::msleep(1000);
92 QThread::msleep(1000);
93 }
93 }
94 }
94 }
95
95
96 void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
96 void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
97 {
97 {
98 if (auto app = sqpApp) {
98 if (auto app = sqpApp) {
99 auto &networkController = app->networkController();
99 auto &networkController = app->networkController();
100 networkController.onReplyCanceled(acqIdentifier);
100 networkController.onReplyCanceled(acqIdentifier);
101 }
101 }
102 }
102 }
103
103
104 void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
104 void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
105 std::shared_ptr<QNetworkRequest> networkRequest,
105 std::shared_ptr<QNetworkRequest> networkRequest,
106 double progress)
106 double progress)
107 {
107 {
108 qCDebug(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
108 qCDebug(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
109 << networkRequest.get() << progress;
109 << networkRequest.get() << progress;
110 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
110 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
111 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
111 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
112
112
113 // Update the progression for the current request
113 // Update the progression for the current request
114 auto requestPtr = networkRequest;
114 auto requestPtr = networkRequest;
115 auto findRequest = [requestPtr](const auto &entry) { return requestPtr == entry.first; };
115 auto findRequest = [requestPtr](const auto &entry) { return requestPtr == entry.first; };
116
116
117 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
117 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
118 auto requestProgressMapEnd = requestProgressMap.end();
118 auto requestProgressMapEnd = requestProgressMap.end();
119 auto requestProgressMapIt
119 auto requestProgressMapIt
120 = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
120 = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
121
121
122 if (requestProgressMapIt != requestProgressMapEnd) {
122 if (requestProgressMapIt != requestProgressMapEnd) {
123 requestProgressMapIt->second = progress;
123 requestProgressMapIt->second = progress;
124 }
124 }
125 else {
125 else {
126 // This case can happened when a progression is send after the request has been
126 // This case can happened when a progression is send after the request has been
127 // finished.
127 // finished.
128 // Generaly the case when aborting a request
128 // Generaly the case when aborting a request
129 qCDebug(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress") << acqIdentifier
129 qCDebug(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress") << acqIdentifier
130 << networkRequest.get() << progress;
130 << networkRequest.get() << progress;
131 }
131 }
132
132
133 // Compute the current final progress and notify it
133 // Compute the current final progress and notify it
134 double finalProgress = 0.0;
134 double finalProgress = 0.0;
135
135
136 auto fraq = requestProgressMap.size();
136 auto fraq = requestProgressMap.size();
137
137
138 for (auto requestProgress : requestProgressMap) {
138 for (auto requestProgress : requestProgressMap) {
139 finalProgress += requestProgress.second;
139 finalProgress += requestProgress.second;
140 qCDebug(LOG_AmdaProvider()) << tr("Current final progress without fraq:")
140 qCDebug(LOG_AmdaProvider()) << tr("Current final progress without fraq:")
141 << finalProgress << requestProgress.second;
141 << finalProgress << requestProgress.second;
142 }
142 }
143
143
144 if (fraq > 0) {
144 if (fraq > 0) {
145 finalProgress = finalProgress / fraq;
145 finalProgress = finalProgress / fraq;
146 }
146 }
147
147
148 qCDebug(LOG_AmdaProvider()) << tr("Current final progress: ") << fraq << finalProgress;
148 qCDebug(LOG_AmdaProvider()) << tr("Current final progress: ") << fraq << finalProgress;
149 emit dataProvidedProgress(acqIdentifier, finalProgress);
149 emit dataProvidedProgress(acqIdentifier, finalProgress);
150 }
150 }
151 else {
151 else {
152 // This case can happened when a progression is send after the request has been finished.
152 // This case can happened when a progression is send after the request has been finished.
153 // Generaly the case when aborting a request
153 // Generaly the case when aborting a request
154 emit dataProvidedProgress(acqIdentifier, 100.0);
154 emit dataProvidedProgress(acqIdentifier, 100.0);
155 }
155 }
156 }
156 }
157
157
158 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
158 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
159 {
159 {
160 // Retrieves product ID from data: if the value is invalid, no request is made
160 // Retrieves product ID from data: if the value is invalid, no request is made
161 auto productId = data.value(AMDA_XML_ID_KEY).toString();
161 auto productId = data.value(AMDA_XML_ID_KEY).toString();
162 if (productId.isNull()) {
162 if (productId.isNull()) {
163 qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
163 qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
164 return;
164 return;
165 }
165 }
166
166
167 // Retrieves the data type that determines whether the expected format for the result file is
167 // Retrieves the data type that determines whether the expected format for the result file is
168 // scalar, vector...
168 // scalar, vector...
169 auto productValueType = valueType(data.value(AMDA_DATA_TYPE_KEY).toString());
169 auto productValueType = valueType(data.value(AMDA_DATA_TYPE_KEY).toString());
170
170
171 // /////////// //
171 // /////////// //
172 // Creates URL //
172 // Creates URL //
173 // /////////// //
173 // /////////// //
174
174
175 auto startDate = dateFormat(dateTime.m_TStart);
175 auto startDate = dateFormat(dateTime.m_TStart);
176 auto endDate = dateFormat(dateTime.m_TEnd);
176 auto endDate = dateFormat(dateTime.m_TEnd);
177
177
178 auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(startDate, endDate, productId)};
178 auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(startDate, endDate, productId)};
179 qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
179 qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
180 auto tempFile = std::make_shared<QTemporaryFile>();
180 auto tempFile = std::make_shared<QTemporaryFile>();
181
181
182 // LAMBDA
182 // LAMBDA
183 auto httpDownloadFinished = [this, dateTime, tempFile,
183 auto httpDownloadFinished = [this, dateTime, tempFile,
184 productValueType](QNetworkReply *reply, QUuid dataId) noexcept {
184 productValueType](QNetworkReply *reply, QUuid dataId) noexcept {
185
185
186 // Don't do anything if the reply was abort
186 // Don't do anything if the reply was abort
187 if (reply->error() == QNetworkReply::NoError) {
187 if (reply->error() == QNetworkReply::NoError) {
188
188
189 if (tempFile) {
189 if (tempFile) {
190 auto replyReadAll = reply->readAll();
190 auto replyReadAll = reply->readAll();
191 if (!replyReadAll.isEmpty()) {
191 if (!replyReadAll.isEmpty()) {
192 tempFile->write(replyReadAll);
192 tempFile->write(replyReadAll);
193 }
193 }
194 tempFile->close();
194 tempFile->close();
195
195
196 // Parse results file
196 // Parse results file
197 if (auto dataSeries
197 if (auto dataSeries
198 = AmdaResultParser::readTxt(tempFile->fileName(), productValueType)) {
198 = AmdaResultParser::readTxt(tempFile->fileName(), productValueType)) {
199 emit dataProvided(dataId, dataSeries, dateTime);
199 emit dataProvided(dataId, dataSeries, dateTime);
200 }
200 }
201 else {
201 else {
202 /// @todo ALX : debug
202 /// @todo ALX : debug
203 emit dataProvidedFailed(dataId);
203 emit dataProvidedFailed(dataId);
204 }
204 }
205 }
205 }
206 qCDebug(LOG_AmdaProvider()) << tr("acquisition requests erase because of finishing")
206 qCDebug(LOG_AmdaProvider()) << tr("acquisition requests erase because of finishing")
207 << dataId;
207 << dataId;
208 m_AcqIdToRequestProgressMap.erase(dataId);
208 m_AcqIdToRequestProgressMap.erase(dataId);
209 }
209 }
210 else {
210 else {
211 qCCritical(LOG_AmdaProvider()) << tr("httpDownloadFinished ERROR");
211 qCCritical(LOG_AmdaProvider()) << tr("httpDownloadFinished ERROR");
212 emit dataProvidedFailed(dataId);
212 emit dataProvidedFailed(dataId);
213 }
213 }
214
214
215 };
215 };
216 auto httpFinishedLambda
216 auto httpFinishedLambda
217 = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
217 = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
218
218
219 // Don't do anything if the reply was abort
219 // Don't do anything if the reply was abort
220 if (reply->error() == QNetworkReply::NoError) {
220 if (reply->error() == QNetworkReply::NoError) {
221 auto downloadFileUrl = QUrl{QString{reply->readAll()}.trimmed()};
221 // AMDA v2: auto downloadFileUrl = QUrl{QString{reply->readAll()}.trimmed()};
222 auto downloadFileUrl = QUrl{QString{reply->readAll()}};
222
223
223 qCInfo(LOG_AmdaProvider())
224 qCInfo(LOG_AmdaProvider())
224 << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
225 << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
225 // Executes request for downloading file //
226 // Executes request for downloading file //
226
227
227 // Creates destination file
228 // Creates destination file
228 if (tempFile->open()) {
229 if (tempFile->open()) {
229 // Executes request and store the request for progression
230 // Executes request and store the request for progression
230 auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
231 auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
231 updateRequestProgress(dataId, request, 0.0);
232 updateRequestProgress(dataId, request, 0.0);
232 emit requestConstructed(request, dataId, httpDownloadFinished);
233 emit requestConstructed(request, dataId, httpDownloadFinished);
233 }
234 }
234 else {
235 else {
235 emit dataProvidedFailed(dataId);
236 emit dataProvidedFailed(dataId);
236 }
237 }
237 }
238 }
238 else {
239 else {
239 qCDebug(LOG_AmdaProvider())
240 qCDebug(LOG_AmdaProvider())
240 << tr("acquisition requests erase because of aborting") << dataId;
241 << tr("acquisition requests erase because of aborting") << dataId;
241 qCCritical(LOG_AmdaProvider()) << tr("httpFinishedLambda ERROR");
242 qCCritical(LOG_AmdaProvider()) << tr("httpFinishedLambda ERROR");
242 m_AcqIdToRequestProgressMap.erase(dataId);
243 m_AcqIdToRequestProgressMap.erase(dataId);
243 emit dataProvidedFailed(dataId);
244 emit dataProvidedFailed(dataId);
244 }
245 }
245 };
246 };
246
247
247 // //////////////// //
248 // //////////////// //
248 // Executes request //
249 // Executes request //
249 // //////////////// //
250 // //////////////// //
250
251
251 auto request = std::make_shared<QNetworkRequest>(url);
252 auto request = std::make_shared<QNetworkRequest>(url);
252 qCDebug(LOG_AmdaProvider()) << tr("First Request creation") << request.get();
253 qCDebug(LOG_AmdaProvider()) << tr("First Request creation") << request.get();
253 updateRequestProgress(token, request, 0.0);
254 updateRequestProgress(token, request, 0.0);
254
255
255 emit requestConstructed(request, token, httpFinishedLambda);
256 emit requestConstructed(request, token, httpFinishedLambda);
256 }
257 }
257
258
258 void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
259 void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
259 std::shared_ptr<QNetworkRequest> request, double progress)
260 std::shared_ptr<QNetworkRequest> request, double progress)
260 {
261 {
261 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
262 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
262 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
263 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
263 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
264 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
264 auto requestProgressMapIt = requestProgressMap.find(request);
265 auto requestProgressMapIt = requestProgressMap.find(request);
265 if (requestProgressMapIt != requestProgressMap.end()) {
266 if (requestProgressMapIt != requestProgressMap.end()) {
266 requestProgressMapIt->second = progress;
267 requestProgressMapIt->second = progress;
267 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new progress for request")
268 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new progress for request")
268 << acqIdentifier << request.get() << progress;
269 << acqIdentifier << request.get() << progress;
269 }
270 }
270 else {
271 else {
271 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new request") << acqIdentifier
272 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new request") << acqIdentifier
272 << request.get() << progress;
273 << request.get() << progress;
273 acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
274 acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
274 }
275 }
275 }
276 }
276 else {
277 else {
277 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new acqIdentifier")
278 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new acqIdentifier")
278 << acqIdentifier << request.get() << progress;
279 << acqIdentifier << request.get() << progress;
279 auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
280 auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
280 requestProgressMap.insert(std::make_pair(request, progress));
281 requestProgressMap.insert(std::make_pair(request, progress));
281 m_AcqIdToRequestProgressMap.insert(
282 m_AcqIdToRequestProgressMap.insert(
282 std::make_pair(acqIdentifier, std::move(requestProgressMap)));
283 std::make_pair(acqIdentifier, std::move(requestProgressMap)));
283 }
284 }
284 }
285 }
General Comments 0
You need to be logged in to leave comments. Login now