##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Updates AMDA provider to handle different server URLs...
Alexandre Leroux -
r943:a2f640a8dec0
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,282 +1,304
1 #include "AmdaProvider.h"
1 #include "AmdaProvider.h"
2 #include "AmdaDefs.h"
2 #include "AmdaDefs.h"
3 #include "AmdaResultParser.h"
3 #include "AmdaResultParser.h"
4
4
5 #include <Common/DateUtils.h>
5 #include <Common/DateUtils.h>
6 #include <Data/DataProviderParameters.h>
6 #include <Data/DataProviderParameters.h>
7 #include <Network/NetworkController.h>
7 #include <Network/NetworkController.h>
8 #include <SqpApplication.h>
8 #include <SqpApplication.h>
9 #include <Variable/Variable.h>
9 #include <Variable/Variable.h>
10
10
11 #include <QNetworkAccessManager>
11 #include <QNetworkAccessManager>
12 #include <QNetworkReply>
12 #include <QNetworkReply>
13 #include <QTemporaryFile>
13 #include <QTemporaryFile>
14 #include <QThread>
14 #include <QThread>
15
15
16 Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")
16 Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")
17
17
18 namespace {
18 namespace {
19
19
20 /// URL of the default AMDA server
21 const auto AMDA_SERVER_URL = QStringLiteral("amda.irap.omp.eu");
22
23 /// URL of the AMDA test server
24 const auto AMDA_TEST_SERVER_URL = QStringLiteral("amdatest.irap.omp.eu");
25
20 /// URL format for a request on AMDA server. The parameters are as follows:
26 /// URL format for a request on AMDA server. The parameters are as follows:
21 /// - %1: start date
27 /// - %1: server URL
22 /// - %2: end date
28 /// - %2: start date
23 /// - %3: parameter id
29 /// - %3: end date
30 /// - %4: parameter id
24 /// AMDA V2: http://amdatest.irap.omp.eu/php/rest/
31 /// AMDA V2: http://amdatest.irap.omp.eu/php/rest/
25 const auto AMDA_URL_FORMAT = QStringLiteral(
32 const auto AMDA_URL_FORMAT = QStringLiteral(
26 "http://amda.irap.omp.eu/php/rest/"
33 "http://%1/php/rest/"
27 "getParameter.php?startTime=%1&stopTime=%2&parameterID=%3&outputFormat=ASCII&"
34 "getParameter.php?startTime=%2&stopTime=%3&parameterID=%4&outputFormat=ASCII&"
28 "timeFormat=ISO8601&gzip=0");
35 "timeFormat=ISO8601&gzip=0");
29
36
30 /// Dates format passed in the URL (e.g 2013-09-23T09:00)
37 /// Dates format passed in the URL (e.g 2013-09-23T09:00)
31 const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
38 const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
32
39
33 /// Formats a time to a date that can be passed in URL
40 /// Formats a time to a date that can be passed in URL
34 QString dateFormat(double sqpRange) noexcept
41 QString dateFormat(double sqpRange) noexcept
35 {
42 {
36 auto dateTime = DateUtils::dateTime(sqpRange);
43 auto dateTime = DateUtils::dateTime(sqpRange);
37 return dateTime.toString(AMDA_TIME_FORMAT);
44 return dateTime.toString(AMDA_TIME_FORMAT);
38 }
45 }
39
46
47 /// Returns the URL of the AMDA server queried for requests, depending on the type of server passed
48 /// as a parameter
49 QString serverURL(const QString &server)
50 {
51 if (server == QString{"amdatest"}) {
52 return AMDA_TEST_SERVER_URL;
53 }
54 else {
55 return AMDA_SERVER_URL;
56 }
57 }
58
40 AmdaResultParser::ValueType valueType(const QString &valueType)
59 AmdaResultParser::ValueType valueType(const QString &valueType)
41 {
60 {
42 if (valueType == QStringLiteral("scalar")) {
61 if (valueType == QStringLiteral("scalar")) {
43 return AmdaResultParser::ValueType::SCALAR;
62 return AmdaResultParser::ValueType::SCALAR;
44 }
63 }
45 else if (valueType == QStringLiteral("vector")) {
64 else if (valueType == QStringLiteral("vector")) {
46 return AmdaResultParser::ValueType::VECTOR;
65 return AmdaResultParser::ValueType::VECTOR;
47 }
66 }
48 else {
67 else {
49 return AmdaResultParser::ValueType::UNKNOWN;
68 return AmdaResultParser::ValueType::UNKNOWN;
50 }
69 }
51 }
70 }
52
71
53 } // namespace
72 } // namespace
54
73
55 AmdaProvider::AmdaProvider()
74 AmdaProvider::AmdaProvider()
56 {
75 {
57 qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
76 qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
58 if (auto app = sqpApp) {
77 if (auto app = sqpApp) {
59 auto &networkController = app->networkController();
78 auto &networkController = app->networkController();
60 connect(this, SIGNAL(requestConstructed(std::shared_ptr<QNetworkRequest>, QUuid,
79 connect(this, SIGNAL(requestConstructed(std::shared_ptr<QNetworkRequest>, QUuid,
61 std::function<void(QNetworkReply *, QUuid)>)),
80 std::function<void(QNetworkReply *, QUuid)>)),
62 &networkController,
81 &networkController,
63 SLOT(onProcessRequested(std::shared_ptr<QNetworkRequest>, QUuid,
82 SLOT(onProcessRequested(std::shared_ptr<QNetworkRequest>, QUuid,
64 std::function<void(QNetworkReply *, QUuid)>)));
83 std::function<void(QNetworkReply *, QUuid)>)));
65
84
66
85
67 connect(&sqpApp->networkController(),
86 connect(&sqpApp->networkController(),
68 SIGNAL(replyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)),
87 SIGNAL(replyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)),
69 this,
88 this,
70 SLOT(onReplyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)));
89 SLOT(onReplyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)));
71 }
90 }
72 }
91 }
73
92
74 std::shared_ptr<IDataProvider> AmdaProvider::clone() const
93 std::shared_ptr<IDataProvider> AmdaProvider::clone() const
75 {
94 {
76 // No copy is made in the clone
95 // No copy is made in the clone
77 return std::make_shared<AmdaProvider>();
96 return std::make_shared<AmdaProvider>();
78 }
97 }
79
98
80 void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderParameters &parameters)
99 void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderParameters &parameters)
81 {
100 {
82 // NOTE: Try to use multithread if possible
101 // NOTE: Try to use multithread if possible
83 const auto times = parameters.m_Times;
102 const auto times = parameters.m_Times;
84 const auto data = parameters.m_Data;
103 const auto data = parameters.m_Data;
85 for (const auto &dateTime : qAsConst(times)) {
104 for (const auto &dateTime : qAsConst(times)) {
86 qCDebug(LOG_AmdaProvider()) << tr("TORM AmdaProvider::requestDataLoading ") << acqIdentifier
105 qCDebug(LOG_AmdaProvider()) << tr("TORM AmdaProvider::requestDataLoading ") << acqIdentifier
87 << dateTime;
106 << dateTime;
88 this->retrieveData(acqIdentifier, dateTime, data);
107 this->retrieveData(acqIdentifier, dateTime, data);
89
108
90
109
91 // TORM when AMDA will support quick asynchrone request
110 // TORM when AMDA will support quick asynchrone request
92 QThread::msleep(1000);
111 QThread::msleep(1000);
93 }
112 }
94 }
113 }
95
114
96 void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
115 void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
97 {
116 {
98 if (auto app = sqpApp) {
117 if (auto app = sqpApp) {
99 auto &networkController = app->networkController();
118 auto &networkController = app->networkController();
100 networkController.onReplyCanceled(acqIdentifier);
119 networkController.onReplyCanceled(acqIdentifier);
101 }
120 }
102 }
121 }
103
122
104 void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
123 void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
105 std::shared_ptr<QNetworkRequest> networkRequest,
124 std::shared_ptr<QNetworkRequest> networkRequest,
106 double progress)
125 double progress)
107 {
126 {
108 qCDebug(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
127 qCDebug(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
109 << networkRequest.get() << progress;
128 << networkRequest.get() << progress;
110 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
129 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
111 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
130 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
112
131
113 // Update the progression for the current request
132 // Update the progression for the current request
114 auto requestPtr = networkRequest;
133 auto requestPtr = networkRequest;
115 auto findRequest = [requestPtr](const auto &entry) { return requestPtr == entry.first; };
134 auto findRequest = [requestPtr](const auto &entry) { return requestPtr == entry.first; };
116
135
117 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
136 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
118 auto requestProgressMapEnd = requestProgressMap.end();
137 auto requestProgressMapEnd = requestProgressMap.end();
119 auto requestProgressMapIt
138 auto requestProgressMapIt
120 = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
139 = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
121
140
122 if (requestProgressMapIt != requestProgressMapEnd) {
141 if (requestProgressMapIt != requestProgressMapEnd) {
123 requestProgressMapIt->second = progress;
142 requestProgressMapIt->second = progress;
124 }
143 }
125 else {
144 else {
126 // This case can happened when a progression is send after the request has been
145 // This case can happened when a progression is send after the request has been
127 // finished.
146 // finished.
128 // Generaly the case when aborting a request
147 // Generaly the case when aborting a request
129 qCDebug(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress") << acqIdentifier
148 qCDebug(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress") << acqIdentifier
130 << networkRequest.get() << progress;
149 << networkRequest.get() << progress;
131 }
150 }
132
151
133 // Compute the current final progress and notify it
152 // Compute the current final progress and notify it
134 double finalProgress = 0.0;
153 double finalProgress = 0.0;
135
154
136 auto fraq = requestProgressMap.size();
155 auto fraq = requestProgressMap.size();
137
156
138 for (auto requestProgress : requestProgressMap) {
157 for (auto requestProgress : requestProgressMap) {
139 finalProgress += requestProgress.second;
158 finalProgress += requestProgress.second;
140 qCDebug(LOG_AmdaProvider()) << tr("Current final progress without fraq:")
159 qCDebug(LOG_AmdaProvider()) << tr("Current final progress without fraq:")
141 << finalProgress << requestProgress.second;
160 << finalProgress << requestProgress.second;
142 }
161 }
143
162
144 if (fraq > 0) {
163 if (fraq > 0) {
145 finalProgress = finalProgress / fraq;
164 finalProgress = finalProgress / fraq;
146 }
165 }
147
166
148 qCDebug(LOG_AmdaProvider()) << tr("Current final progress: ") << fraq << finalProgress;
167 qCDebug(LOG_AmdaProvider()) << tr("Current final progress: ") << fraq << finalProgress;
149 emit dataProvidedProgress(acqIdentifier, finalProgress);
168 emit dataProvidedProgress(acqIdentifier, finalProgress);
150 }
169 }
151 else {
170 else {
152 // This case can happened when a progression is send after the request has been finished.
171 // This case can happened when a progression is send after the request has been finished.
153 // Generaly the case when aborting a request
172 // Generaly the case when aborting a request
154 emit dataProvidedProgress(acqIdentifier, 100.0);
173 emit dataProvidedProgress(acqIdentifier, 100.0);
155 }
174 }
156 }
175 }
157
176
158 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
177 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
159 {
178 {
160 // Retrieves product ID from data: if the value is invalid, no request is made
179 // Retrieves product ID from data: if the value is invalid, no request is made
161 auto productId = data.value(AMDA_XML_ID_KEY).toString();
180 auto productId = data.value(AMDA_XML_ID_KEY).toString();
162 if (productId.isNull()) {
181 if (productId.isNull()) {
163 qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
182 qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
164 return;
183 return;
165 }
184 }
166
185
167 // Retrieves the data type that determines whether the expected format for the result file is
186 // Retrieves the data type that determines whether the expected format for the result file is
168 // scalar, vector...
187 // scalar, vector...
169 auto productValueType = valueType(data.value(AMDA_DATA_TYPE_KEY).toString());
188 auto productValueType = valueType(data.value(AMDA_DATA_TYPE_KEY).toString());
170
189
190 // Gets the server being queried to retrieve the product. It's then used to set the server URL
191 auto productServer = data.value(AMDA_SERVER_KEY).toString();
192
171 // /////////// //
193 // /////////// //
172 // Creates URL //
194 // Creates URL //
173 // /////////// //
195 // /////////// //
174
196
175 auto startDate = dateFormat(dateTime.m_TStart);
197 auto startDate = dateFormat(dateTime.m_TStart);
176 auto endDate = dateFormat(dateTime.m_TEnd);
198 auto endDate = dateFormat(dateTime.m_TEnd);
177
199
178 auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(startDate, endDate, productId)};
200 auto url = QUrl{
201 QString{AMDA_URL_FORMAT}.arg(serverURL(productServer), startDate, endDate, productId)};
179 qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
202 qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
180 auto tempFile = std::make_shared<QTemporaryFile>();
203 auto tempFile = std::make_shared<QTemporaryFile>();
181
204
182 // LAMBDA
205 // LAMBDA
183 auto httpDownloadFinished = [this, dateTime, tempFile,
206 auto httpDownloadFinished = [this, dateTime, tempFile,
184 productValueType](QNetworkReply *reply, QUuid dataId) noexcept {
207 productValueType](QNetworkReply *reply, QUuid dataId) noexcept {
185
208
186 // Don't do anything if the reply was abort
209 // Don't do anything if the reply was abort
187 if (reply->error() == QNetworkReply::NoError) {
210 if (reply->error() == QNetworkReply::NoError) {
188
211
189 if (tempFile) {
212 if (tempFile) {
190 auto replyReadAll = reply->readAll();
213 auto replyReadAll = reply->readAll();
191 if (!replyReadAll.isEmpty()) {
214 if (!replyReadAll.isEmpty()) {
192 tempFile->write(replyReadAll);
215 tempFile->write(replyReadAll);
193 }
216 }
194 tempFile->close();
217 tempFile->close();
195
218
196 // Parse results file
219 // Parse results file
197 if (auto dataSeries
220 if (auto dataSeries
198 = AmdaResultParser::readTxt(tempFile->fileName(), productValueType)) {
221 = AmdaResultParser::readTxt(tempFile->fileName(), productValueType)) {
199 emit dataProvided(dataId, dataSeries, dateTime);
222 emit dataProvided(dataId, dataSeries, dateTime);
200 }
223 }
201 else {
224 else {
202 /// @todo ALX : debug
225 /// @todo ALX : debug
203 emit dataProvidedFailed(dataId);
226 emit dataProvidedFailed(dataId);
204 }
227 }
205 }
228 }
206 m_AcqIdToRequestProgressMap.erase(dataId);
229 m_AcqIdToRequestProgressMap.erase(dataId);
207 }
230 }
208 else {
231 else {
209 qCCritical(LOG_AmdaProvider()) << tr("httpDownloadFinished ERROR");
232 qCCritical(LOG_AmdaProvider()) << tr("httpDownloadFinished ERROR");
210 emit dataProvidedFailed(dataId);
233 emit dataProvidedFailed(dataId);
211 }
234 }
212
235
213 };
236 };
214 auto httpFinishedLambda
237 auto httpFinishedLambda
215 = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
238 = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
216
239
217 // Don't do anything if the reply was abort
240 // Don't do anything if the reply was abort
218 if (reply->error() == QNetworkReply::NoError) {
241 if (reply->error() == QNetworkReply::NoError) {
219 // AMDA v2: auto downloadFileUrl = QUrl{QString{reply->readAll()}.trimmed()};
242 auto downloadFileUrl = QUrl{QString{reply->readAll()}.trimmed()};
220 auto downloadFileUrl = QUrl{QString{reply->readAll()}};
221
243
222 qCInfo(LOG_AmdaProvider())
244 qCInfo(LOG_AmdaProvider())
223 << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
245 << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
224 // Executes request for downloading file //
246 // Executes request for downloading file //
225
247
226 // Creates destination file
248 // Creates destination file
227 if (tempFile->open()) {
249 if (tempFile->open()) {
228 // Executes request and store the request for progression
250 // Executes request and store the request for progression
229 auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
251 auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
230 updateRequestProgress(dataId, request, 0.0);
252 updateRequestProgress(dataId, request, 0.0);
231 emit requestConstructed(request, dataId, httpDownloadFinished);
253 emit requestConstructed(request, dataId, httpDownloadFinished);
232 }
254 }
233 else {
255 else {
234 emit dataProvidedFailed(dataId);
256 emit dataProvidedFailed(dataId);
235 }
257 }
236 }
258 }
237 else {
259 else {
238 qCCritical(LOG_AmdaProvider()) << tr("httpFinishedLambda ERROR");
260 qCCritical(LOG_AmdaProvider()) << tr("httpFinishedLambda ERROR");
239 m_AcqIdToRequestProgressMap.erase(dataId);
261 m_AcqIdToRequestProgressMap.erase(dataId);
240 emit dataProvidedFailed(dataId);
262 emit dataProvidedFailed(dataId);
241 }
263 }
242 };
264 };
243
265
244 // //////////////// //
266 // //////////////// //
245 // Executes request //
267 // Executes request //
246 // //////////////// //
268 // //////////////// //
247
269
248 auto request = std::make_shared<QNetworkRequest>(url);
270 auto request = std::make_shared<QNetworkRequest>(url);
249 qCDebug(LOG_AmdaProvider()) << tr("First Request creation") << request.get();
271 qCDebug(LOG_AmdaProvider()) << tr("First Request creation") << request.get();
250 updateRequestProgress(token, request, 0.0);
272 updateRequestProgress(token, request, 0.0);
251
273
252 emit requestConstructed(request, token, httpFinishedLambda);
274 emit requestConstructed(request, token, httpFinishedLambda);
253 }
275 }
254
276
255 void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
277 void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
256 std::shared_ptr<QNetworkRequest> request, double progress)
278 std::shared_ptr<QNetworkRequest> request, double progress)
257 {
279 {
258 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress request") << request.get();
280 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress request") << request.get();
259 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
281 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
260 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
282 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
261 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
283 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
262 auto requestProgressMapIt = requestProgressMap.find(request);
284 auto requestProgressMapIt = requestProgressMap.find(request);
263 if (requestProgressMapIt != requestProgressMap.end()) {
285 if (requestProgressMapIt != requestProgressMap.end()) {
264 requestProgressMapIt->second = progress;
286 requestProgressMapIt->second = progress;
265 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new progress for request")
287 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new progress for request")
266 << acqIdentifier << request.get() << progress;
288 << acqIdentifier << request.get() << progress;
267 }
289 }
268 else {
290 else {
269 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new request") << acqIdentifier
291 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new request") << acqIdentifier
270 << request.get() << progress;
292 << request.get() << progress;
271 acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
293 acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
272 }
294 }
273 }
295 }
274 else {
296 else {
275 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new acqIdentifier")
297 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new acqIdentifier")
276 << acqIdentifier << request.get() << progress;
298 << acqIdentifier << request.get() << progress;
277 auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
299 auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
278 requestProgressMap.insert(std::make_pair(request, progress));
300 requestProgressMap.insert(std::make_pair(request, progress));
279 m_AcqIdToRequestProgressMap.insert(
301 m_AcqIdToRequestProgressMap.insert(
280 std::make_pair(acqIdentifier, std::move(requestProgressMap)));
302 std::make_pair(acqIdentifier, std::move(requestProgressMap)));
281 }
303 }
282 }
304 }
General Comments 0
You need to be logged in to leave comments. Login now