##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Changes AMDA url
Alexandre Leroux -
r774:837a0a824da1
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,283 +1,284
1 #include "AmdaProvider.h"
1 #include "AmdaProvider.h"
2 #include "AmdaDefs.h"
2 #include "AmdaDefs.h"
3 #include "AmdaResultParser.h"
3 #include "AmdaResultParser.h"
4
4
5 #include <Common/DateUtils.h>
5 #include <Common/DateUtils.h>
6 #include <Data/DataProviderParameters.h>
6 #include <Data/DataProviderParameters.h>
7 #include <Network/NetworkController.h>
7 #include <Network/NetworkController.h>
8 #include <SqpApplication.h>
8 #include <SqpApplication.h>
9 #include <Variable/Variable.h>
9 #include <Variable/Variable.h>
10
10
11 #include <QNetworkAccessManager>
11 #include <QNetworkAccessManager>
12 #include <QNetworkReply>
12 #include <QNetworkReply>
13 #include <QTemporaryFile>
13 #include <QTemporaryFile>
14 #include <QThread>
14 #include <QThread>
15
15
16 Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")
16 Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")
17
17
18 namespace {
18 namespace {
19
19
20 /// URL format for a request on AMDA server. The parameters are as follows:
20 /// URL format for a request on AMDA server. The parameters are as follows:
21 /// - %1: start date
21 /// - %1: start date
22 /// - %2: end date
22 /// - %2: end date
23 /// - %3: parameter id
23 /// - %3: parameter id
24 /// Old url: http://amda.irap.omp.eu/php/rest/
24 const auto AMDA_URL_FORMAT = QStringLiteral(
25 const auto AMDA_URL_FORMAT = QStringLiteral(
25 "http://amda.irap.omp.eu/php/rest/"
26 "http://amdatest.irap.omp.eu/php/rest/"
26 "getParameter.php?startTime=%1&stopTime=%2&parameterID=%3&outputFormat=ASCII&"
27 "getParameter.php?startTime=%1&stopTime=%2&parameterID=%3&outputFormat=ASCII&"
27 "timeFormat=ISO8601&gzip=0");
28 "timeFormat=ISO8601&gzip=0");
28
29
29 /// Dates format passed in the URL (e.g 2013-09-23T09:00)
30 /// Dates format passed in the URL (e.g 2013-09-23T09:00)
30 const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
31 const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
31
32
32 /// Formats a time to a date that can be passed in URL
33 /// Formats a time to a date that can be passed in URL
33 QString dateFormat(double sqpRange) noexcept
34 QString dateFormat(double sqpRange) noexcept
34 {
35 {
35 auto dateTime = DateUtils::dateTime(sqpRange);
36 auto dateTime = DateUtils::dateTime(sqpRange);
36 return dateTime.toString(AMDA_TIME_FORMAT);
37 return dateTime.toString(AMDA_TIME_FORMAT);
37 }
38 }
38
39
39 AmdaResultParser::ValueType valueType(const QString &valueType)
40 AmdaResultParser::ValueType valueType(const QString &valueType)
40 {
41 {
41 if (valueType == QStringLiteral("scalar")) {
42 if (valueType == QStringLiteral("scalar")) {
42 return AmdaResultParser::ValueType::SCALAR;
43 return AmdaResultParser::ValueType::SCALAR;
43 }
44 }
44 else if (valueType == QStringLiteral("vector")) {
45 else if (valueType == QStringLiteral("vector")) {
45 return AmdaResultParser::ValueType::VECTOR;
46 return AmdaResultParser::ValueType::VECTOR;
46 }
47 }
47 else {
48 else {
48 return AmdaResultParser::ValueType::UNKNOWN;
49 return AmdaResultParser::ValueType::UNKNOWN;
49 }
50 }
50 }
51 }
51
52
52 } // namespace
53 } // namespace
53
54
54 AmdaProvider::AmdaProvider()
55 AmdaProvider::AmdaProvider()
55 {
56 {
56 qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
57 qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
57 if (auto app = sqpApp) {
58 if (auto app = sqpApp) {
58 auto &networkController = app->networkController();
59 auto &networkController = app->networkController();
59 connect(this, SIGNAL(requestConstructed(std::shared_ptr<QNetworkRequest>, QUuid,
60 connect(this, SIGNAL(requestConstructed(std::shared_ptr<QNetworkRequest>, QUuid,
60 std::function<void(QNetworkReply *, QUuid)>)),
61 std::function<void(QNetworkReply *, QUuid)>)),
61 &networkController,
62 &networkController,
62 SLOT(onProcessRequested(std::shared_ptr<QNetworkRequest>, QUuid,
63 SLOT(onProcessRequested(std::shared_ptr<QNetworkRequest>, QUuid,
63 std::function<void(QNetworkReply *, QUuid)>)));
64 std::function<void(QNetworkReply *, QUuid)>)));
64
65
65
66
66 connect(&sqpApp->networkController(),
67 connect(&sqpApp->networkController(),
67 SIGNAL(replyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)),
68 SIGNAL(replyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)),
68 this,
69 this,
69 SLOT(onReplyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)));
70 SLOT(onReplyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)));
70 }
71 }
71 }
72 }
72
73
73 std::shared_ptr<IDataProvider> AmdaProvider::clone() const
74 std::shared_ptr<IDataProvider> AmdaProvider::clone() const
74 {
75 {
75 // No copy is made in the clone
76 // No copy is made in the clone
76 return std::make_shared<AmdaProvider>();
77 return std::make_shared<AmdaProvider>();
77 }
78 }
78
79
79 void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderParameters &parameters)
80 void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderParameters &parameters)
80 {
81 {
81 // NOTE: Try to use multithread if possible
82 // NOTE: Try to use multithread if possible
82 const auto times = parameters.m_Times;
83 const auto times = parameters.m_Times;
83 const auto data = parameters.m_Data;
84 const auto data = parameters.m_Data;
84 for (const auto &dateTime : qAsConst(times)) {
85 for (const auto &dateTime : qAsConst(times)) {
85 qCDebug(LOG_AmdaProvider()) << tr("TORM AmdaProvider::requestDataLoading ") << acqIdentifier
86 qCDebug(LOG_AmdaProvider()) << tr("TORM AmdaProvider::requestDataLoading ") << acqIdentifier
86 << dateTime;
87 << dateTime;
87 this->retrieveData(acqIdentifier, dateTime, data);
88 this->retrieveData(acqIdentifier, dateTime, data);
88
89
89
90
90 // TORM when AMDA will support quick asynchrone request
91 // TORM when AMDA will support quick asynchrone request
91 QThread::msleep(1000);
92 QThread::msleep(1000);
92 }
93 }
93 }
94 }
94
95
95 void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
96 void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
96 {
97 {
97 if (auto app = sqpApp) {
98 if (auto app = sqpApp) {
98 auto &networkController = app->networkController();
99 auto &networkController = app->networkController();
99 networkController.onReplyCanceled(acqIdentifier);
100 networkController.onReplyCanceled(acqIdentifier);
100 }
101 }
101 }
102 }
102
103
103 void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
104 void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
104 std::shared_ptr<QNetworkRequest> networkRequest,
105 std::shared_ptr<QNetworkRequest> networkRequest,
105 double progress)
106 double progress)
106 {
107 {
107 qCDebug(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
108 qCDebug(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
108 << networkRequest.get() << progress;
109 << networkRequest.get() << progress;
109 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
110 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
110 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
111 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
111
112
112 // Update the progression for the current request
113 // Update the progression for the current request
113 auto requestPtr = networkRequest;
114 auto requestPtr = networkRequest;
114 auto findRequest = [requestPtr](const auto &entry) { return requestPtr == entry.first; };
115 auto findRequest = [requestPtr](const auto &entry) { return requestPtr == entry.first; };
115
116
116 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
117 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
117 auto requestProgressMapEnd = requestProgressMap.end();
118 auto requestProgressMapEnd = requestProgressMap.end();
118 auto requestProgressMapIt
119 auto requestProgressMapIt
119 = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
120 = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
120
121
121 if (requestProgressMapIt != requestProgressMapEnd) {
122 if (requestProgressMapIt != requestProgressMapEnd) {
122 requestProgressMapIt->second = progress;
123 requestProgressMapIt->second = progress;
123 }
124 }
124 else {
125 else {
125 // This case can happened when a progression is send after the request has been
126 // This case can happened when a progression is send after the request has been
126 // finished.
127 // finished.
127 // Generaly the case when aborting a request
128 // Generaly the case when aborting a request
128 qCDebug(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress") << acqIdentifier
129 qCDebug(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress") << acqIdentifier
129 << networkRequest.get() << progress;
130 << networkRequest.get() << progress;
130 }
131 }
131
132
132 // Compute the current final progress and notify it
133 // Compute the current final progress and notify it
133 double finalProgress = 0.0;
134 double finalProgress = 0.0;
134
135
135 auto fraq = requestProgressMap.size();
136 auto fraq = requestProgressMap.size();
136
137
137 for (auto requestProgress : requestProgressMap) {
138 for (auto requestProgress : requestProgressMap) {
138 finalProgress += requestProgress.second;
139 finalProgress += requestProgress.second;
139 qCDebug(LOG_AmdaProvider()) << tr("Current final progress without fraq:")
140 qCDebug(LOG_AmdaProvider()) << tr("Current final progress without fraq:")
140 << finalProgress << requestProgress.second;
141 << finalProgress << requestProgress.second;
141 }
142 }
142
143
143 if (fraq > 0) {
144 if (fraq > 0) {
144 finalProgress = finalProgress / fraq;
145 finalProgress = finalProgress / fraq;
145 }
146 }
146
147
147 qCDebug(LOG_AmdaProvider()) << tr("Current final progress: ") << fraq << finalProgress;
148 qCDebug(LOG_AmdaProvider()) << tr("Current final progress: ") << fraq << finalProgress;
148 emit dataProvidedProgress(acqIdentifier, finalProgress);
149 emit dataProvidedProgress(acqIdentifier, finalProgress);
149 }
150 }
150 else {
151 else {
151 // This case can happened when a progression is send after the request has been finished.
152 // This case can happened when a progression is send after the request has been finished.
152 // Generaly the case when aborting a request
153 // Generaly the case when aborting a request
153 emit dataProvidedProgress(acqIdentifier, 100.0);
154 emit dataProvidedProgress(acqIdentifier, 100.0);
154 }
155 }
155 }
156 }
156
157
157 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
158 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
158 {
159 {
159 // Retrieves product ID from data: if the value is invalid, no request is made
160 // Retrieves product ID from data: if the value is invalid, no request is made
160 auto productId = data.value(AMDA_XML_ID_KEY).toString();
161 auto productId = data.value(AMDA_XML_ID_KEY).toString();
161 if (productId.isNull()) {
162 if (productId.isNull()) {
162 qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
163 qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
163 return;
164 return;
164 }
165 }
165
166
166 // Retrieves the data type that determines whether the expected format for the result file is
167 // Retrieves the data type that determines whether the expected format for the result file is
167 // scalar, vector...
168 // scalar, vector...
168 auto productValueType = valueType(data.value(AMDA_DATA_TYPE_KEY).toString());
169 auto productValueType = valueType(data.value(AMDA_DATA_TYPE_KEY).toString());
169
170
170 // /////////// //
171 // /////////// //
171 // Creates URL //
172 // Creates URL //
172 // /////////// //
173 // /////////// //
173
174
174 auto startDate = dateFormat(dateTime.m_TStart);
175 auto startDate = dateFormat(dateTime.m_TStart);
175 auto endDate = dateFormat(dateTime.m_TEnd);
176 auto endDate = dateFormat(dateTime.m_TEnd);
176
177
177 auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(startDate, endDate, productId)};
178 auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(startDate, endDate, productId)};
178 qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
179 qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
179 auto tempFile = std::make_shared<QTemporaryFile>();
180 auto tempFile = std::make_shared<QTemporaryFile>();
180
181
181 // LAMBDA
182 // LAMBDA
182 auto httpDownloadFinished = [this, dateTime, tempFile,
183 auto httpDownloadFinished = [this, dateTime, tempFile,
183 productValueType](QNetworkReply *reply, QUuid dataId) noexcept {
184 productValueType](QNetworkReply *reply, QUuid dataId) noexcept {
184
185
185 // Don't do anything if the reply was abort
186 // Don't do anything if the reply was abort
186 if (reply->error() == QNetworkReply::NoError) {
187 if (reply->error() == QNetworkReply::NoError) {
187
188
188 if (tempFile) {
189 if (tempFile) {
189 auto replyReadAll = reply->readAll();
190 auto replyReadAll = reply->readAll();
190 if (!replyReadAll.isEmpty()) {
191 if (!replyReadAll.isEmpty()) {
191 tempFile->write(replyReadAll);
192 tempFile->write(replyReadAll);
192 }
193 }
193 tempFile->close();
194 tempFile->close();
194
195
195 // Parse results file
196 // Parse results file
196 if (auto dataSeries
197 if (auto dataSeries
197 = AmdaResultParser::readTxt(tempFile->fileName(), productValueType)) {
198 = AmdaResultParser::readTxt(tempFile->fileName(), productValueType)) {
198 emit dataProvided(dataId, dataSeries, dateTime);
199 emit dataProvided(dataId, dataSeries, dateTime);
199 }
200 }
200 else {
201 else {
201 /// @todo ALX : debug
202 /// @todo ALX : debug
202 emit dataProvidedFailed(dataId);
203 emit dataProvidedFailed(dataId);
203 }
204 }
204 }
205 }
205 qCDebug(LOG_AmdaProvider()) << tr("acquisition requests erase because of finishing")
206 qCDebug(LOG_AmdaProvider()) << tr("acquisition requests erase because of finishing")
206 << dataId;
207 << dataId;
207 m_AcqIdToRequestProgressMap.erase(dataId);
208 m_AcqIdToRequestProgressMap.erase(dataId);
208 }
209 }
209 else {
210 else {
210 qCCritical(LOG_AmdaProvider()) << tr("httpDownloadFinished ERROR");
211 qCCritical(LOG_AmdaProvider()) << tr("httpDownloadFinished ERROR");
211 emit dataProvidedFailed(dataId);
212 emit dataProvidedFailed(dataId);
212 }
213 }
213
214
214 };
215 };
215 auto httpFinishedLambda
216 auto httpFinishedLambda
216 = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
217 = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
217
218
218 // Don't do anything if the reply was abort
219 // Don't do anything if the reply was abort
219 if (reply->error() == QNetworkReply::NoError) {
220 if (reply->error() == QNetworkReply::NoError) {
220 auto downloadFileUrl = QUrl{QString{reply->readAll()}};
221 auto downloadFileUrl = QUrl{QString{reply->readAll()}.trimmed()};
221
222
222 qCInfo(LOG_AmdaProvider())
223 qCInfo(LOG_AmdaProvider())
223 << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
224 << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
224 // Executes request for downloading file //
225 // Executes request for downloading file //
225
226
226 // Creates destination file
227 // Creates destination file
227 if (tempFile->open()) {
228 if (tempFile->open()) {
228 // Executes request and store the request for progression
229 // Executes request and store the request for progression
229 auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
230 auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
230 updateRequestProgress(dataId, request, 0.0);
231 updateRequestProgress(dataId, request, 0.0);
231 emit requestConstructed(request, dataId, httpDownloadFinished);
232 emit requestConstructed(request, dataId, httpDownloadFinished);
232 }
233 }
233 else {
234 else {
234 emit dataProvidedFailed(dataId);
235 emit dataProvidedFailed(dataId);
235 }
236 }
236 }
237 }
237 else {
238 else {
238 qCDebug(LOG_AmdaProvider())
239 qCDebug(LOG_AmdaProvider())
239 << tr("acquisition requests erase because of aborting") << dataId;
240 << tr("acquisition requests erase because of aborting") << dataId;
240 qCCritical(LOG_AmdaProvider()) << tr("httpFinishedLambda ERROR");
241 qCCritical(LOG_AmdaProvider()) << tr("httpFinishedLambda ERROR");
241 m_AcqIdToRequestProgressMap.erase(dataId);
242 m_AcqIdToRequestProgressMap.erase(dataId);
242 emit dataProvidedFailed(dataId);
243 emit dataProvidedFailed(dataId);
243 }
244 }
244 };
245 };
245
246
246 // //////////////// //
247 // //////////////// //
247 // Executes request //
248 // Executes request //
248 // //////////////// //
249 // //////////////// //
249
250
250 auto request = std::make_shared<QNetworkRequest>(url);
251 auto request = std::make_shared<QNetworkRequest>(url);
251 qCDebug(LOG_AmdaProvider()) << tr("First Request creation") << request.get();
252 qCDebug(LOG_AmdaProvider()) << tr("First Request creation") << request.get();
252 updateRequestProgress(token, request, 0.0);
253 updateRequestProgress(token, request, 0.0);
253
254
254 emit requestConstructed(request, token, httpFinishedLambda);
255 emit requestConstructed(request, token, httpFinishedLambda);
255 }
256 }
256
257
257 void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
258 void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
258 std::shared_ptr<QNetworkRequest> request, double progress)
259 std::shared_ptr<QNetworkRequest> request, double progress)
259 {
260 {
260 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
261 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
261 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
262 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
262 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
263 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
263 auto requestProgressMapIt = requestProgressMap.find(request);
264 auto requestProgressMapIt = requestProgressMap.find(request);
264 if (requestProgressMapIt != requestProgressMap.end()) {
265 if (requestProgressMapIt != requestProgressMap.end()) {
265 requestProgressMapIt->second = progress;
266 requestProgressMapIt->second = progress;
266 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new progress for request")
267 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new progress for request")
267 << acqIdentifier << request.get() << progress;
268 << acqIdentifier << request.get() << progress;
268 }
269 }
269 else {
270 else {
270 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new request") << acqIdentifier
271 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new request") << acqIdentifier
271 << request.get() << progress;
272 << request.get() << progress;
272 acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
273 acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
273 }
274 }
274 }
275 }
275 else {
276 else {
276 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new acqIdentifier")
277 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new acqIdentifier")
277 << acqIdentifier << request.get() << progress;
278 << acqIdentifier << request.get() << progress;
278 auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
279 auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
279 requestProgressMap.insert(std::make_pair(request, progress));
280 requestProgressMap.insert(std::make_pair(request, progress));
280 m_AcqIdToRequestProgressMap.insert(
281 m_AcqIdToRequestProgressMap.insert(
281 std::make_pair(acqIdentifier, std::move(requestProgressMap)));
282 std::make_pair(acqIdentifier, std::move(requestProgressMap)));
282 }
283 }
283 }
284 }
General Comments 0
You need to be logged in to leave comments. Login now