##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Merge branch 'feature/AmdaV2' into develop
Alexandre Leroux -
r779:72250e9134d9 merge
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,3
1 Not Found
2
3 The requested URL /AMDA/data/WSRESULT/imf(0)-1343153090-1343153092-60.txt was not found on this server. No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,6
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
4 2013-09-23T09:00:30.000 NaN
5 2013-09-23T09:01:30.000 -2.71850
6 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,6
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
4 NaN -3.01425
5 2013-09-23T09:01:30.000 -2.71850
6 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,2
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,6
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
4 2013-09-23T09:00:30.000 -2.83950 1.05141 3.01547
5 2013-09-23T09:01:30.000 -2.71850
6 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,13
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
4 2013-09-23T09:00:30.000 -2.83950
5 2013-09-23T09:01:30.000 -2.71850
6 2013-09-23T09:02:30.000 -2.52150
7 2013-09-23T09:03:30.000 -2.57633
8 2013-09-23T09:04:30.000 -2.58050
9 2013-09-23T09:05:30.000 -2.48325
10 2013-09-23T09:06:30.000 -2.63025
11 2013-09-23T09:07:30.000 -2.55800
12 2013-09-23T09:08:30.000 -2.43250
13 2013-09-23T09:09:30.000 -2.42200 No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,12
1 #Time Format : YYYY-MM-DDThh:mm:ss.mls
2 #imf - Type : Local Parameter @ CDPP/AMDA - Name : imf_gse - Units : nT - Size : 3 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
3 2013-07-02T09:13:50.000 -0.332000 3.20600 0.0580000
4 2013-07-02T09:14:06.000 -1.01100 2.99900 0.496000
5 2013-07-02T09:14:22.000 -1.45700 2.78500 1.01800
6 2013-07-02T09:14:38.000 -1.29300 2.73600 1.48500
7 2013-07-02T09:14:54.000 -1.21700 2.61200 1.66200
8 2013-07-02T09:15:10.000 -1.44300 2.56400 1.50500
9 2013-07-02T09:15:26.000 -1.27800 2.89200 1.16800
10 2013-07-02T09:15:42.000 -1.20200 2.86200 1.24400
11 2013-07-02T09:15:58.000 -1.22000 2.85900 1.15000
12 2013-07-02T09:16:14.000 -1.25900 2.76400 1.35800 No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,6
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
4 23/09/2013 07:50:30 -2.83950
5 2013-09-23T09:01:30.000 -2.71850
6 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,6
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #Wrong unit comment
4 2013-09-23T09:00:30.000 -2.83950
5 2013-09-23T09:01:30.000 -2.71850
6 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -0,0 +1,6
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
4 2013-09-23T09:00:30.000 abc
5 2013-09-23T09:01:30.000 -2.71850
6 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,63 +1,62
1 1
2 2 core_moc_headers = [
3 3 'include/Data/IDataProvider.h',
4 4 'include/DataSource/DataSourceController.h',
5 5 'include/DataSource/DataSourceItemAction.h',
6 6 'include/Network/NetworkController.h',
7 7 'include/Time/TimeController.h',
8 8 'include/Variable/Variable.h',
9 9 'include/Variable/VariableCacheController.h',
10 10 'include/Variable/VariableController.h',
11 11 'include/Variable/VariableAcquisitionWorker.h',
12 12 'include/Variable/VariableCacheStrategy.h',
13 13 'include/Variable/VariableSynchronizationGroup.h',
14 14 'include/Variable/VariableModel.h',
15 15 'include/Visualization/VisualizationController.h'
16 16 ]
17 17
18 18
19 19 core_moc_files = qt5.preprocess(moc_headers : core_moc_headers)
20 20
21 21 core_sources = [
22 22 'src/Common/DateUtils.cpp',
23 23 'src/Common/StringUtils.cpp',
24 24 'src/Data/ScalarSeries.cpp',
25 25 'src/Data/DataSeriesIterator.cpp',
26 26 'src/Data/ArrayDataIterator.cpp',
27 27 'src/Data/VectorSeries.cpp',
28 28 'src/DataSource/DataSourceController.cpp',
29 29 'src/DataSource/DataSourceItem.cpp',
30 30 'src/DataSource/DataSourceItemAction.cpp',
31 31 'src/Network/NetworkController.cpp',
32 32 'src/Plugin/PluginManager.cpp',
33 33 'src/Settings/SqpSettingsDefs.cpp',
34 34 'src/Time/TimeController.cpp',
35 35 'src/Variable/Variable.cpp',
36 36 'src/Variable/VariableCacheController.cpp',
37 37 'src/Variable/VariableController.cpp',
38 38 'src/Variable/VariableAcquisitionWorker.cpp',
39 'src/Variable/VariableCacheStrategy.cpp',
40 39 'src/Variable/VariableSynchronizationGroup.cpp',
41 40 'src/Variable/VariableModel.cpp',
42 41 'src/Visualization/VisualizationController.cpp'
43 42 ]
44 43
45 44 core_inc = include_directories(['include', '../plugin/include'])
46 45
47 46 sciqlop_core_lib = library('sciqlopcore',
48 47 core_sources,
49 48 core_moc_files,
50 49 cpp_args : '-DCORE_LIB',
51 50 include_directories : core_inc,
52 51 dependencies : [qt5core, qt5network],
53 52 install : true
54 53 )
55 54
56 55
57 56 sciqlop_core = declare_dependency(link_with : sciqlop_core_lib,
58 57 include_directories : core_inc,
59 58 dependencies : [qt5core, qt5network])
60 59
61 60
62 61 subdir('tests')
63 62
Show file before | Show file after | Hide comments
@@ -1,283 +1,284
1 1 #include "AmdaProvider.h"
2 2 #include "AmdaDefs.h"
3 3 #include "AmdaResultParser.h"
4 4
5 5 #include <Common/DateUtils.h>
6 6 #include <Data/DataProviderParameters.h>
7 7 #include <Network/NetworkController.h>
8 8 #include <SqpApplication.h>
9 9 #include <Variable/Variable.h>
10 10
11 11 #include <QNetworkAccessManager>
12 12 #include <QNetworkReply>
13 13 #include <QTemporaryFile>
14 14 #include <QThread>
15 15
16 16 Q_LOGGING_CATEGORY(LOG_AmdaProvider, "AmdaProvider")
17 17
18 18 namespace {
19 19
20 20 /// URL format for a request on AMDA server. The parameters are as follows:
21 21 /// - %1: start date
22 22 /// - %2: end date
23 23 /// - %3: parameter id
24 /// Old url: http://amda.irap.omp.eu/php/rest/
24 25 const auto AMDA_URL_FORMAT = QStringLiteral(
25 "http://amda.irap.omp.eu/php/rest/"
26 "http://amdatest.irap.omp.eu/php/rest/"
26 27 "getParameter.php?startTime=%1&stopTime=%2&parameterID=%3&outputFormat=ASCII&"
27 28 "timeFormat=ISO8601&gzip=0");
28 29
29 30 /// Dates format passed in the URL (e.g 2013-09-23T09:00)
30 31 const auto AMDA_TIME_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss");
31 32
32 33 /// Formats a time to a date that can be passed in URL
33 34 QString dateFormat(double sqpRange) noexcept
34 35 {
35 36 auto dateTime = DateUtils::dateTime(sqpRange);
36 37 return dateTime.toString(AMDA_TIME_FORMAT);
37 38 }
38 39
39 40 AmdaResultParser::ValueType valueType(const QString &valueType)
40 41 {
41 42 if (valueType == QStringLiteral("scalar")) {
42 43 return AmdaResultParser::ValueType::SCALAR;
43 44 }
44 45 else if (valueType == QStringLiteral("vector")) {
45 46 return AmdaResultParser::ValueType::VECTOR;
46 47 }
47 48 else {
48 49 return AmdaResultParser::ValueType::UNKNOWN;
49 50 }
50 51 }
51 52
52 53 } // namespace
53 54
54 55 AmdaProvider::AmdaProvider()
55 56 {
56 57 qCDebug(LOG_AmdaProvider()) << tr("AmdaProvider::AmdaProvider") << QThread::currentThread();
57 58 if (auto app = sqpApp) {
58 59 auto &networkController = app->networkController();
59 60 connect(this, SIGNAL(requestConstructed(std::shared_ptr<QNetworkRequest>, QUuid,
60 61 std::function<void(QNetworkReply *, QUuid)>)),
61 62 &networkController,
62 63 SLOT(onProcessRequested(std::shared_ptr<QNetworkRequest>, QUuid,
63 64 std::function<void(QNetworkReply *, QUuid)>)));
64 65
65 66
66 67 connect(&sqpApp->networkController(),
67 68 SIGNAL(replyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)),
68 69 this,
69 70 SLOT(onReplyDownloadProgress(QUuid, std::shared_ptr<QNetworkRequest>, double)));
70 71 }
71 72 }
72 73
73 74 std::shared_ptr<IDataProvider> AmdaProvider::clone() const
74 75 {
75 76 // No copy is made in the clone
76 77 return std::make_shared<AmdaProvider>();
77 78 }
78 79
79 80 void AmdaProvider::requestDataLoading(QUuid acqIdentifier, const DataProviderParameters &parameters)
80 81 {
81 82 // NOTE: Try to use multithread if possible
82 83 const auto times = parameters.m_Times;
83 84 const auto data = parameters.m_Data;
84 85 for (const auto &dateTime : qAsConst(times)) {
85 86 qCDebug(LOG_AmdaProvider()) << tr("TORM AmdaProvider::requestDataLoading ") << acqIdentifier
86 87 << dateTime;
87 88 this->retrieveData(acqIdentifier, dateTime, data);
88 89
89 90
90 91 // TORM when AMDA will support quick asynchrone request
91 92 QThread::msleep(1000);
92 93 }
93 94 }
94 95
95 96 void AmdaProvider::requestDataAborting(QUuid acqIdentifier)
96 97 {
97 98 if (auto app = sqpApp) {
98 99 auto &networkController = app->networkController();
99 100 networkController.onReplyCanceled(acqIdentifier);
100 101 }
101 102 }
102 103
103 104 void AmdaProvider::onReplyDownloadProgress(QUuid acqIdentifier,
104 105 std::shared_ptr<QNetworkRequest> networkRequest,
105 106 double progress)
106 107 {
107 108 qCDebug(LOG_AmdaProvider()) << tr("onReplyDownloadProgress") << acqIdentifier
108 109 << networkRequest.get() << progress;
109 110 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
110 111 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
111 112
112 113 // Update the progression for the current request
113 114 auto requestPtr = networkRequest;
114 115 auto findRequest = [requestPtr](const auto &entry) { return requestPtr == entry.first; };
115 116
116 117 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
117 118 auto requestProgressMapEnd = requestProgressMap.end();
118 119 auto requestProgressMapIt
119 120 = std::find_if(requestProgressMap.begin(), requestProgressMapEnd, findRequest);
120 121
121 122 if (requestProgressMapIt != requestProgressMapEnd) {
122 123 requestProgressMapIt->second = progress;
123 124 }
124 125 else {
125 126 // This case can happened when a progression is send after the request has been
126 127 // finished.
127 128 // Generaly the case when aborting a request
128 129 qCDebug(LOG_AmdaProvider()) << tr("Can't retrieve Request in progress") << acqIdentifier
129 130 << networkRequest.get() << progress;
130 131 }
131 132
132 133 // Compute the current final progress and notify it
133 134 double finalProgress = 0.0;
134 135
135 136 auto fraq = requestProgressMap.size();
136 137
137 138 for (auto requestProgress : requestProgressMap) {
138 139 finalProgress += requestProgress.second;
139 140 qCDebug(LOG_AmdaProvider()) << tr("Current final progress without fraq:")
140 141 << finalProgress << requestProgress.second;
141 142 }
142 143
143 144 if (fraq > 0) {
144 145 finalProgress = finalProgress / fraq;
145 146 }
146 147
147 148 qCDebug(LOG_AmdaProvider()) << tr("Current final progress: ") << fraq << finalProgress;
148 149 emit dataProvidedProgress(acqIdentifier, finalProgress);
149 150 }
150 151 else {
151 152 // This case can happened when a progression is send after the request has been finished.
152 153 // Generaly the case when aborting a request
153 154 emit dataProvidedProgress(acqIdentifier, 100.0);
154 155 }
155 156 }
156 157
157 158 void AmdaProvider::retrieveData(QUuid token, const SqpRange &dateTime, const QVariantHash &data)
158 159 {
159 160 // Retrieves product ID from data: if the value is invalid, no request is made
160 161 auto productId = data.value(AMDA_XML_ID_KEY).toString();
161 162 if (productId.isNull()) {
162 163 qCCritical(LOG_AmdaProvider()) << tr("Can't retrieve data: unknown product id");
163 164 return;
164 165 }
165 166
166 167 // Retrieves the data type that determines whether the expected format for the result file is
167 168 // scalar, vector...
168 169 auto productValueType = valueType(data.value(AMDA_DATA_TYPE_KEY).toString());
169 170
170 171 // /////////// //
171 172 // Creates URL //
172 173 // /////////// //
173 174
174 175 auto startDate = dateFormat(dateTime.m_TStart);
175 176 auto endDate = dateFormat(dateTime.m_TEnd);
176 177
177 178 auto url = QUrl{QString{AMDA_URL_FORMAT}.arg(startDate, endDate, productId)};
178 179 qCInfo(LOG_AmdaProvider()) << tr("TORM AmdaProvider::retrieveData url:") << url;
179 180 auto tempFile = std::make_shared<QTemporaryFile>();
180 181
181 182 // LAMBDA
182 183 auto httpDownloadFinished = [this, dateTime, tempFile,
183 184 productValueType](QNetworkReply *reply, QUuid dataId) noexcept {
184 185
185 186 // Don't do anything if the reply was abort
186 187 if (reply->error() == QNetworkReply::NoError) {
187 188
188 189 if (tempFile) {
189 190 auto replyReadAll = reply->readAll();
190 191 if (!replyReadAll.isEmpty()) {
191 192 tempFile->write(replyReadAll);
192 193 }
193 194 tempFile->close();
194 195
195 196 // Parse results file
196 197 if (auto dataSeries
197 198 = AmdaResultParser::readTxt(tempFile->fileName(), productValueType)) {
198 199 emit dataProvided(dataId, dataSeries, dateTime);
199 200 }
200 201 else {
201 202 /// @todo ALX : debug
202 203 emit dataProvidedFailed(dataId);
203 204 }
204 205 }
205 206 qCDebug(LOG_AmdaProvider()) << tr("acquisition requests erase because of finishing")
206 207 << dataId;
207 208 m_AcqIdToRequestProgressMap.erase(dataId);
208 209 }
209 210 else {
210 211 qCCritical(LOG_AmdaProvider()) << tr("httpDownloadFinished ERROR");
211 212 emit dataProvidedFailed(dataId);
212 213 }
213 214
214 215 };
215 216 auto httpFinishedLambda
216 217 = [this, httpDownloadFinished, tempFile](QNetworkReply *reply, QUuid dataId) noexcept {
217 218
218 219 // Don't do anything if the reply was abort
219 220 if (reply->error() == QNetworkReply::NoError) {
220 auto downloadFileUrl = QUrl{QString{reply->readAll()}};
221 auto downloadFileUrl = QUrl{QString{reply->readAll()}.trimmed()};
221 222
222 223 qCInfo(LOG_AmdaProvider())
223 224 << tr("TORM AmdaProvider::retrieveData downloadFileUrl:") << downloadFileUrl;
224 225 // Executes request for downloading file //
225 226
226 227 // Creates destination file
227 228 if (tempFile->open()) {
228 229 // Executes request and store the request for progression
229 230 auto request = std::make_shared<QNetworkRequest>(downloadFileUrl);
230 231 updateRequestProgress(dataId, request, 0.0);
231 232 emit requestConstructed(request, dataId, httpDownloadFinished);
232 233 }
233 234 else {
234 235 emit dataProvidedFailed(dataId);
235 236 }
236 237 }
237 238 else {
238 239 qCDebug(LOG_AmdaProvider())
239 240 << tr("acquisition requests erase because of aborting") << dataId;
240 241 qCCritical(LOG_AmdaProvider()) << tr("httpFinishedLambda ERROR");
241 242 m_AcqIdToRequestProgressMap.erase(dataId);
242 243 emit dataProvidedFailed(dataId);
243 244 }
244 245 };
245 246
246 247 // //////////////// //
247 248 // Executes request //
248 249 // //////////////// //
249 250
250 251 auto request = std::make_shared<QNetworkRequest>(url);
251 252 qCDebug(LOG_AmdaProvider()) << tr("First Request creation") << request.get();
252 253 updateRequestProgress(token, request, 0.0);
253 254
254 255 emit requestConstructed(request, token, httpFinishedLambda);
255 256 }
256 257
257 258 void AmdaProvider::updateRequestProgress(QUuid acqIdentifier,
258 259 std::shared_ptr<QNetworkRequest> request, double progress)
259 260 {
260 261 auto acqIdToRequestProgressMapIt = m_AcqIdToRequestProgressMap.find(acqIdentifier);
261 262 if (acqIdToRequestProgressMapIt != m_AcqIdToRequestProgressMap.end()) {
262 263 auto &requestProgressMap = acqIdToRequestProgressMapIt->second;
263 264 auto requestProgressMapIt = requestProgressMap.find(request);
264 265 if (requestProgressMapIt != requestProgressMap.end()) {
265 266 requestProgressMapIt->second = progress;
266 267 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new progress for request")
267 268 << acqIdentifier << request.get() << progress;
268 269 }
269 270 else {
270 271 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new request") << acqIdentifier
271 272 << request.get() << progress;
272 273 acqIdToRequestProgressMapIt->second.insert(std::make_pair(request, progress));
273 274 }
274 275 }
275 276 else {
276 277 qCDebug(LOG_AmdaProvider()) << tr("updateRequestProgress new acqIdentifier")
277 278 << acqIdentifier << request.get() << progress;
278 279 auto requestProgressMap = std::map<std::shared_ptr<QNetworkRequest>, double>{};
279 280 requestProgressMap.insert(std::make_pair(request, progress));
280 281 m_AcqIdToRequestProgressMap.insert(
281 282 std::make_pair(acqIdentifier, std::move(requestProgressMap)));
282 283 }
283 284 }
Show file before | Show file after | Hide comments
@@ -1,209 +1,225
1 1 #include "AmdaResultParser.h"
2 2
3 3 #include <Common/DateUtils.h>
4 4 #include <Data/ScalarSeries.h>
5 5 #include <Data/VectorSeries.h>
6 6
7 7 #include <QDateTime>
8 8 #include <QFile>
9 9 #include <QRegularExpression>
10 10
11 11 #include <cmath>
12 12
13 13 Q_LOGGING_CATEGORY(LOG_AmdaResultParser, "AmdaResultParser")
14 14
15 15 namespace {
16 16
17 17 /// Message in result file when the file was not found on server
18 18 const auto FILE_NOT_FOUND_MESSAGE = QStringLiteral("Not Found");
19 19
20 /// Format for dates in result files
21 const auto DATE_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss.zzz");
22
23 20 /// Separator between values in a result line
24 21 const auto RESULT_LINE_SEPARATOR = QRegularExpression{QStringLiteral("\\s+")};
25 22
23 /// Regex to find the header of the data in the file. This header indicates the end of comments in
24 /// the file
25 const auto DATA_HEADER_REGEX = QRegularExpression{QStringLiteral("#\\s*DATA\\s*:")};
26
27 /// Format for dates in result files
28 const auto DATE_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss.zzz");
29
26 30 /// Regex to find unit in a line. Examples of valid lines:
27 /// ... - Units : nT - ...
28 /// ... -Units:nT- ...
29 /// ... -Units: mΒ²- ...
30 /// ... - Units : m/s - ...
31 const auto UNIT_REGEX = QRegularExpression{QStringLiteral("-\\s*Units\\s*:\\s*(.+?)\\s*-")};
31 /// ... PARAMETER_UNITS : nT ...
32 /// ... PARAMETER_UNITS:nT ...
33 /// ... PARAMETER_UNITS: mΒ² ...
34 /// ... PARAMETER_UNITS : m/s ...
35 const auto UNIT_REGEX = QRegularExpression{QStringLiteral("\\s*PARAMETER_UNITS\\s*:\\s*(.+)")};
36
37 QDateTime dateTimeFromString(const QString &stringDate) noexcept
38 {
39 #if QT_VERSION >= QT_VERSION_CHECK(5, 8, 0)
40 return QDateTime::fromString(stringDate, Qt::ISODateWithMs);
41 #else
42 return QDateTime::fromString(stringDate, DATE_FORMAT);
43 #endif
44 }
32 45
33 46 /// Converts a string date to a double date
34 47 /// @return a double that represents the date in seconds, NaN if the string date can't be converted
35 48 double doubleDate(const QString &stringDate) noexcept
36 49 {
37 auto dateTime = QDateTime::fromString(stringDate, DATE_FORMAT);
50 // Format: yyyy-MM-ddThh:mm:ss.zzz
51 auto dateTime = dateTimeFromString(stringDate);
38 52 dateTime.setTimeSpec(Qt::UTC);
39 53 return dateTime.isValid() ? DateUtils::secondsSinceEpoch(dateTime)
40 54 : std::numeric_limits<double>::quiet_NaN();
41 55 }
42 56
43 57 /// Checks if a line is a comment line
44 58 bool isCommentLine(const QString &line)
45 59 {
46 60 return line.startsWith("#");
47 61 }
48 62
49 63 /// @return the number of lines to be read depending on the type of value passed in parameter
50 64 int nbValues(AmdaResultParser::ValueType valueType) noexcept
51 65 {
52 66 switch (valueType) {
53 67 case AmdaResultParser::ValueType::SCALAR:
54 68 return 1;
55 69 case AmdaResultParser::ValueType::VECTOR:
56 70 return 3;
57 71 case AmdaResultParser::ValueType::UNKNOWN:
58 72 // Invalid case
59 73 break;
60 74 }
61 75
62 76 // Invalid cases
63 77 qCCritical(LOG_AmdaResultParser())
64 78 << QObject::tr("Can't get the number of values to read: unsupported type");
65 79 return 0;
66 80 }
67 81
68 82 /**
69 83 * Reads stream to retrieve x-axis unit
70 84 * @param stream the stream to read
71 85 * @return the unit that has been read in the stream, a default unit (time unit with no label) if an
72 86 * error occured during reading
73 87 */
74 88 Unit readXAxisUnit(QTextStream &stream)
75 89 {
76 90 QString line{};
77 91
78 // Searches unit in the comment lines
79 while (stream.readLineInto(&line) && isCommentLine(line)) {
92 // Searches unit in the comment lines (as long as the reading has not reached the data header)
93 while (stream.readLineInto(&line) && !line.contains(DATA_HEADER_REGEX)) {
80 94 auto match = UNIT_REGEX.match(line);
81 95 if (match.hasMatch()) {
82 96 return Unit{match.captured(1), true};
83 97 }
84 98 }
85 99
86 100 qCWarning(LOG_AmdaResultParser()) << QObject::tr("The unit could not be found in the file");
87 101
88 102 // Error cases
89 103 return Unit{{}, true};
90 104 }
91 105
92 106 /**
93 107 * Reads stream to retrieve results
94 108 * @param stream the stream to read
95 109 * @return the pair of vectors x-axis data/values data that has been read in the stream
96 110 */
97 111 std::pair<std::vector<double>, std::vector<double> >
98 112 readResults(QTextStream &stream, AmdaResultParser::ValueType valueType)
99 113 {
100 auto expectedNbValues = nbValues(valueType);
114 auto expectedNbValues = nbValues(valueType) + 1;
101 115
102 116 auto xData = std::vector<double>{};
103 117 auto valuesData = std::vector<double>{};
104 118
105 119 QString line{};
106 120
107 while (stream.readLineInto(&line)) {
108 // Ignore comment lines
109 if (!isCommentLine(line)) {
121 // Skip comment lines
122 while (stream.readLineInto(&line) && isCommentLine(line)) {
123 }
124
125 if (!stream.atEnd()) {
126 do {
110 127 auto lineData = line.split(RESULT_LINE_SEPARATOR, QString::SkipEmptyParts);
111 if (lineData.size() == expectedNbValues + 1) {
128 if (lineData.size() == expectedNbValues) {
112 129 // X : the data is converted from date to double (in secs)
113 130 auto x = doubleDate(lineData.at(0));
114 131
115 132 // Adds result only if x is valid. Then, if value is invalid, it is set to NaN
116 133 if (!std::isnan(x)) {
117 134 xData.push_back(x);
118 135
119 136 // Values
120 for (auto valueIndex = 0; valueIndex < expectedNbValues; ++valueIndex) {
121 auto column = valueIndex + 1;
137 for (auto valueIndex = 1; valueIndex < expectedNbValues; ++valueIndex) {
138 auto column = valueIndex;
122 139
123 140 bool valueOk;
124 141 auto value = lineData.at(column).toDouble(&valueOk);
125 142
126 143 if (!valueOk) {
127 144 qCWarning(LOG_AmdaResultParser())
128 145 << QObject::tr(
129 146 "Value from (line %1, column %2) is invalid and will be "
130 147 "converted to NaN")
131 148 .arg(line, column);
132 149 value = std::numeric_limits<double>::quiet_NaN();
133 150 }
134 151 valuesData.push_back(value);
135 152 }
136 153 }
137 154 else {
138 155 qCWarning(LOG_AmdaResultParser())
139 156 << QObject::tr("Can't retrieve results from line %1: x is invalid")
140 157 .arg(line);
141 158 }
142 159 }
143 160 else {
144 161 qCWarning(LOG_AmdaResultParser())
145 162 << QObject::tr("Can't retrieve results from line %1: invalid line").arg(line);
146 163 }
147 }
164 } while (stream.readLineInto(&line));
148 165 }
149 166
150 167 return std::make_pair(std::move(xData), std::move(valuesData));
151 168 }
152 169
153 170 } // namespace
154 171
155 172 std::shared_ptr<IDataSeries> AmdaResultParser::readTxt(const QString &filePath,
156 173 ValueType valueType) noexcept
157 174 {
158 175 if (valueType == ValueType::UNKNOWN) {
159 176 qCCritical(LOG_AmdaResultParser())
160 177 << QObject::tr("Can't retrieve AMDA data: the type of values to be read is unknown");
161 178 return nullptr;
162 179 }
163 180
164 181 QFile file{filePath};
165 182
166 183 if (!file.open(QFile::ReadOnly | QIODevice::Text)) {
167 184 qCCritical(LOG_AmdaResultParser())
168 185 << QObject::tr("Can't retrieve AMDA data from file %1: %2")
169 186 .arg(filePath, file.errorString());
170 187 return nullptr;
171 188 }
172 189
173 190 QTextStream stream{&file};
174 191
175 192 // Checks if the file was found on the server
176 193 auto firstLine = stream.readLine();
177 194 if (firstLine.compare(FILE_NOT_FOUND_MESSAGE) == 0) {
178 195 qCCritical(LOG_AmdaResultParser())
179 196 << QObject::tr("Can't retrieve AMDA data from file %1: file was not found on server")
180 197 .arg(filePath);
181 198 return nullptr;
182 199 }
183 200
184 201 // Reads x-axis unit
185 202 stream.seek(0); // returns to the beginning of the file
186 203 auto xAxisUnit = readXAxisUnit(stream);
187 204
188 205 // Reads results
189 stream.seek(0); // returns to the beginning of the file
190 206 auto results = readResults(stream, valueType);
191 207
192 208 // Creates data series
193 209 switch (valueType) {
194 210 case ValueType::SCALAR:
195 211 return std::make_shared<ScalarSeries>(std::move(results.first),
196 212 std::move(results.second), xAxisUnit, Unit{});
197 213 case ValueType::VECTOR:
198 214 return std::make_shared<VectorSeries>(std::move(results.first),
199 215 std::move(results.second), xAxisUnit, Unit{});
200 216 case ValueType::UNKNOWN:
201 217 // Invalid case
202 218 break;
203 219 }
204 220
205 221 // Invalid cases
206 222 qCCritical(LOG_AmdaResultParser())
207 223 << QObject::tr("Can't create data series: unsupported value type");
208 224 return nullptr;
209 225 }
Show file before | Show file after | Hide comments
@@ -1,6 +1,64
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
1 # -----------
2 # AMDA INFO :
3 # -----------
4 # AMDA_ABOUT : Created by CDPP/AMDA(c)
5 # AMDA_VERSION : 3.5.0
6 # AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team
7 #
8 # --------------
9 # REQUEST INFO :
10 # --------------
11 # REQUEST_STRUCTURE : one-file-per-parameter-per-interval
12 # REQUEST_TIME_FORMAT : ISO 8601
13 # REQUEST_OUTPUT_PARAMS : imf
14 #
15 # -----------------
16 # BASE PARAMETERS :
17 # -----------------
18 #
19 # MISSION_ID : NONE
20 #
21 # INSTRUMENT_ID : NONE
22 #
23 # DATASET_ID : ace-imf-all
24 # DATASET_NAME : final / prelim
25 # DATASET_DESCRIPTION : Interplanetary Magnetic Field 16-sec Level2/PRELIM Data
26 # DATASET_SOURCE : CDPP/DDServer
27 # DATASET_GLOBAL_START : 1997-09-02T00:00:12.000
28 # DATASET_GLOBAL_STOP : 2017-09-16T23:59:57.000
29 # DATASET_MIN_SAMPLING : 16
30 # DATASET_MAX_SAMPLING : 16
31 # DATASET_CAVEATS :
32 The quality of ACE level 2 data is such that it is suitable for serious scientific study. However, to avoid confusion and misunderstanding, it is recommended that users consult with the appropriate ACE team members before publishing work derived from the data. The ACE team has worked hard to ensure that the level 2 data are free from errors, but the team cannot accept responsibility for erroneous data, or for misunderstandings about how the data may be used. This is especially true if the appropriate ACE team members are not consulted before publication. At the very least, preprints should be forwarded to the ACE team before publication.
33
34 # DATASET_ACKNOWLEDGEMENT :
35 Please acknowledge the ACE/MAG instrument team and the ACE Science Center
36
37 #
38 # PARAMETER_ID : imf
39 # PARAMETER_NAME : imf
40 # PARAMETER_SHORT_NAME : b_gse
41 # PARAMETER_COMPONENTS : bx,by,bz
42 # PARAMETER_UNITS : nT
43 # PARAMETER_COORDINATE_SYSTEM : GSE
44 # PARAMETER_TENSOR_ORDER : 0
45 # PARAMETER_SI_CONVERSION : 1e-9>T
46 # PARAMETER_TABLE : None
47 # PARAMETER_FILL_VALUE : nan
48 # PARAMETER_UCD : phys.magField
49 #
50 #
51 # ---------------
52 # INTERVAL INFO :
53 # ---------------
54 # INTERVAL_START : 2013-09-23T08:58:12.000
55 # INTERVAL_STOP : 2013-09-23T09:11:48.000
56 #
57 # ------
58 # DATA :
59 # ------
60 # DATA_COLUMNS : AMDA_TIME, imf[0]
61 #
4 62 2013-09-23T09:00:30.000 NaN
5 63 2013-09-23T09:01:30.000 -2.71850
6 64 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,6 +1,64
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
1 # -----------
2 # AMDA INFO :
3 # -----------
4 # AMDA_ABOUT : Created by CDPP/AMDA(c)
5 # AMDA_VERSION : 3.5.0
6 # AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team
7 #
8 # --------------
9 # REQUEST INFO :
10 # --------------
11 # REQUEST_STRUCTURE : one-file-per-parameter-per-interval
12 # REQUEST_TIME_FORMAT : ISO 8601
13 # REQUEST_OUTPUT_PARAMS : imf
14 #
15 # -----------------
16 # BASE PARAMETERS :
17 # -----------------
18 #
19 # MISSION_ID : NONE
20 #
21 # INSTRUMENT_ID : NONE
22 #
23 # DATASET_ID : ace-imf-all
24 # DATASET_NAME : final / prelim
25 # DATASET_DESCRIPTION : Interplanetary Magnetic Field 16-sec Level2/PRELIM Data
26 # DATASET_SOURCE : CDPP/DDServer
27 # DATASET_GLOBAL_START : 1997-09-02T00:00:12.000
28 # DATASET_GLOBAL_STOP : 2017-09-16T23:59:57.000
29 # DATASET_MIN_SAMPLING : 16
30 # DATASET_MAX_SAMPLING : 16
31 # DATASET_CAVEATS :
32 The quality of ACE level 2 data is such that it is suitable for serious scientific study. However, to avoid confusion and misunderstanding, it is recommended that users consult with the appropriate ACE team members before publishing work derived from the data. The ACE team has worked hard to ensure that the level 2 data are free from errors, but the team cannot accept responsibility for erroneous data, or for misunderstandings about how the data may be used. This is especially true if the appropriate ACE team members are not consulted before publication. At the very least, preprints should be forwarded to the ACE team before publication.
33
34 # DATASET_ACKNOWLEDGEMENT :
35 Please acknowledge the ACE/MAG instrument team and the ACE Science Center
36
37 #
38 # PARAMETER_ID : imf
39 # PARAMETER_NAME : imf
40 # PARAMETER_SHORT_NAME : b_gse
41 # PARAMETER_COMPONENTS : bx,by,bz
42 # PARAMETER_UNITS : nT
43 # PARAMETER_COORDINATE_SYSTEM : GSE
44 # PARAMETER_TENSOR_ORDER : 0
45 # PARAMETER_SI_CONVERSION : 1e-9>T
46 # PARAMETER_TABLE : None
47 # PARAMETER_FILL_VALUE : nan
48 # PARAMETER_UCD : phys.magField
49 #
50 #
51 # ---------------
52 # INTERVAL INFO :
53 # ---------------
54 # INTERVAL_START : 2013-09-23T08:58:12.000
55 # INTERVAL_STOP : 2013-09-23T09:11:48.000
56 #
57 # ------
58 # DATA :
59 # ------
60 # DATA_COLUMNS : AMDA_TIME, imf[0]
61 #
4 62 NaN -3.01425
5 63 2013-09-23T09:01:30.000 -2.71850
6 64 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,2 +1,60
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls No newline at end of file
1 # -----------
2 # AMDA INFO :
3 # -----------
4 # AMDA_ABOUT : Created by CDPP/AMDA(c)
5 # AMDA_VERSION : 3.5.0
6 # AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team
7 #
8 # --------------
9 # REQUEST INFO :
10 # --------------
11 # REQUEST_STRUCTURE : one-file-per-parameter-per-interval
12 # REQUEST_TIME_FORMAT : ISO 8601
13 # REQUEST_OUTPUT_PARAMS : imf
14 #
15 # -----------------
16 # BASE PARAMETERS :
17 # -----------------
18 #
19 # MISSION_ID : NONE
20 #
21 # INSTRUMENT_ID : NONE
22 #
23 # DATASET_ID : ace-imf-all
24 # DATASET_NAME : final / prelim
25 # DATASET_DESCRIPTION : Interplanetary Magnetic Field 16-sec Level2/PRELIM Data
26 # DATASET_SOURCE : CDPP/DDServer
27 # DATASET_GLOBAL_START : 1997-09-02T00:00:12.000
28 # DATASET_GLOBAL_STOP : 2017-09-16T23:59:57.000
29 # DATASET_MIN_SAMPLING : 16
30 # DATASET_MAX_SAMPLING : 16
31 # DATASET_CAVEATS :
32 The quality of ACE level 2 data is such that it is suitable for serious scientific study. However, to avoid confusion and misunderstanding, it is recommended that users consult with the appropriate ACE team members before publishing work derived from the data. The ACE team has worked hard to ensure that the level 2 data are free from errors, but the team cannot accept responsibility for erroneous data, or for misunderstandings about how the data may be used. This is especially true if the appropriate ACE team members are not consulted before publication. At the very least, preprints should be forwarded to the ACE team before publication.
33
34 # DATASET_ACKNOWLEDGEMENT :
35 Please acknowledge the ACE/MAG instrument team and the ACE Science Center
36
37 #
38 # PARAMETER_ID : imf
39 # PARAMETER_NAME : imf
40 # PARAMETER_SHORT_NAME : b_gse
41 # PARAMETER_COMPONENTS : bx,by,bz
42 # PARAMETER_COORDINATE_SYSTEM : GSE
43 # PARAMETER_TENSOR_ORDER : 0
44 # PARAMETER_SI_CONVERSION : 1e-9>T
45 # PARAMETER_TABLE : None
46 # PARAMETER_FILL_VALUE : nan
47 # PARAMETER_UCD : phys.magField
48 #
49 #
50 # ---------------
51 # INTERVAL INFO :
52 # ---------------
53 # INTERVAL_START : 2013-09-23T08:58:12.000
54 # INTERVAL_STOP : 2013-09-23T09:11:48.000
55 #
56 # ------
57 # DATA :
58 # ------
59 # DATA_COLUMNS : AMDA_TIME, imf[0]
60 # No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,6 +1,64
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
1 # -----------
2 # AMDA INFO :
3 # -----------
4 # AMDA_ABOUT : Created by CDPP/AMDA(c)
5 # AMDA_VERSION : 3.5.0
6 # AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team
7 #
8 # --------------
9 # REQUEST INFO :
10 # --------------
11 # REQUEST_STRUCTURE : one-file-per-parameter-per-interval
12 # REQUEST_TIME_FORMAT : ISO 8601
13 # REQUEST_OUTPUT_PARAMS : imf
14 #
15 # -----------------
16 # BASE PARAMETERS :
17 # -----------------
18 #
19 # MISSION_ID : NONE
20 #
21 # INSTRUMENT_ID : NONE
22 #
23 # DATASET_ID : ace-imf-all
24 # DATASET_NAME : final / prelim
25 # DATASET_DESCRIPTION : Interplanetary Magnetic Field 16-sec Level2/PRELIM Data
26 # DATASET_SOURCE : CDPP/DDServer
27 # DATASET_GLOBAL_START : 1997-09-02T00:00:12.000
28 # DATASET_GLOBAL_STOP : 2017-09-16T23:59:57.000
29 # DATASET_MIN_SAMPLING : 16
30 # DATASET_MAX_SAMPLING : 16
31 # DATASET_CAVEATS :
32 The quality of ACE level 2 data is such that it is suitable for serious scientific study. However, to avoid confusion and misunderstanding, it is recommended that users consult with the appropriate ACE team members before publishing work derived from the data. The ACE team has worked hard to ensure that the level 2 data are free from errors, but the team cannot accept responsibility for erroneous data, or for misunderstandings about how the data may be used. This is especially true if the appropriate ACE team members are not consulted before publication. At the very least, preprints should be forwarded to the ACE team before publication.
33
34 # DATASET_ACKNOWLEDGEMENT :
35 Please acknowledge the ACE/MAG instrument team and the ACE Science Center
36
37 #
38 # PARAMETER_ID : imf
39 # PARAMETER_NAME : imf
40 # PARAMETER_SHORT_NAME : b_gse
41 # PARAMETER_COMPONENTS : bx,by,bz
42 # PARAMETER_UNITS : nT
43 # PARAMETER_COORDINATE_SYSTEM : GSE
44 # PARAMETER_TENSOR_ORDER : 0
45 # PARAMETER_SI_CONVERSION : 1e-9>T
46 # PARAMETER_TABLE : None
47 # PARAMETER_FILL_VALUE : nan
48 # PARAMETER_UCD : phys.magField
49 #
50 #
51 # ---------------
52 # INTERVAL INFO :
53 # ---------------
54 # INTERVAL_START : 2013-09-23T08:58:12.000
55 # INTERVAL_STOP : 2013-09-23T09:11:48.000
56 #
57 # ------
58 # DATA :
59 # ------
60 # DATA_COLUMNS : AMDA_TIME, imf[0]
61 #
4 62 2013-09-23T09:00:30.000 -2.83950 1.05141 3.01547
5 63 2013-09-23T09:01:30.000 -2.71850
6 64 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,13 +1,71
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
1 # -----------
2 # AMDA INFO :
3 # -----------
4 # AMDA_ABOUT : Created by CDPP/AMDA(c)
5 # AMDA_VERSION : 3.5.0
6 # AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team
7 #
8 # --------------
9 # REQUEST INFO :
10 # --------------
11 # REQUEST_STRUCTURE : one-file-per-parameter-per-interval
12 # REQUEST_TIME_FORMAT : ISO 8601
13 # REQUEST_OUTPUT_PARAMS : imf
14 #
15 # -----------------
16 # BASE PARAMETERS :
17 # -----------------
18 #
19 # MISSION_ID : NONE
20 #
21 # INSTRUMENT_ID : NONE
22 #
23 # DATASET_ID : ace-imf-all
24 # DATASET_NAME : final / prelim
25 # DATASET_DESCRIPTION : Interplanetary Magnetic Field 16-sec Level2/PRELIM Data
26 # DATASET_SOURCE : CDPP/DDServer
27 # DATASET_GLOBAL_START : 1997-09-02T00:00:12.000
28 # DATASET_GLOBAL_STOP : 2017-09-16T23:59:57.000
29 # DATASET_MIN_SAMPLING : 16
30 # DATASET_MAX_SAMPLING : 16
31 # DATASET_CAVEATS :
32 The quality of ACE level 2 data is such that it is suitable for serious scientific study. However, to avoid confusion and misunderstanding, it is recommended that users consult with the appropriate ACE team members before publishing work derived from the data. The ACE team has worked hard to ensure that the level 2 data are free from errors, but the team cannot accept responsibility for erroneous data, or for misunderstandings about how the data may be used. This is especially true if the appropriate ACE team members are not consulted before publication. At the very least, preprints should be forwarded to the ACE team before publication.
33
34 # DATASET_ACKNOWLEDGEMENT :
35 Please acknowledge the ACE/MAG instrument team and the ACE Science Center
36
37 #
38 # PARAMETER_ID : imf
39 # PARAMETER_NAME : imf
40 # PARAMETER_SHORT_NAME : b_gse
41 # PARAMETER_COMPONENTS : bx,by,bz
42 # PARAMETER_UNITS : nT
43 # PARAMETER_COORDINATE_SYSTEM : GSE
44 # PARAMETER_TENSOR_ORDER : 0
45 # PARAMETER_SI_CONVERSION : 1e-9>T
46 # PARAMETER_TABLE : None
47 # PARAMETER_FILL_VALUE : nan
48 # PARAMETER_UCD : phys.magField
49 #
50 #
51 # ---------------
52 # INTERVAL INFO :
53 # ---------------
54 # INTERVAL_START : 2013-09-23T08:58:12.000
55 # INTERVAL_STOP : 2013-09-23T09:11:48.000
56 #
57 # ------
58 # DATA :
59 # ------
60 # DATA_COLUMNS : AMDA_TIME, imf[0]
61 #
4 62 2013-09-23T09:00:30.000 -2.83950
5 63 2013-09-23T09:01:30.000 -2.71850
6 64 2013-09-23T09:02:30.000 -2.52150
7 65 2013-09-23T09:03:30.000 -2.57633
8 66 2013-09-23T09:04:30.000 -2.58050
9 67 2013-09-23T09:05:30.000 -2.48325
10 68 2013-09-23T09:06:30.000 -2.63025
11 69 2013-09-23T09:07:30.000 -2.55800
12 70 2013-09-23T09:08:30.000 -2.43250
13 71 2013-09-23T09:09:30.000 -2.42200 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,12 +1,71
1 #Time Format : YYYY-MM-DDThh:mm:ss.mls
2 #imf - Type : Local Parameter @ CDPP/AMDA - Name : imf_gse - Units : nT - Size : 3 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
1 # -----------
2 # AMDA INFO :
3 # -----------
4 # AMDA_ABOUT : Created by CDPP/AMDA(c)
5 # AMDA_VERSION : 3.5.0
6 # AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team
7 #
8 # --------------
9 # REQUEST INFO :
10 # --------------
11 # REQUEST_STRUCTURE : one-file-per-parameter-per-interval
12 # REQUEST_TIME_FORMAT : ISO 8601
13 # REQUEST_OUTPUT_PARAMS : imf
14 #
15 # -----------------
16 # BASE PARAMETERS :
17 # -----------------
18 #
19 # MISSION_ID : NONE
20 #
21 # INSTRUMENT_ID : NONE
22 #
23 # DATASET_ID : ace-imf-all
24 # DATASET_NAME : final / prelim
25 # DATASET_DESCRIPTION : Interplanetary Magnetic Field 16-sec Level2/PRELIM Data
26 # DATASET_SOURCE : CDPP/DDServer
27 # DATASET_GLOBAL_START : 1997-09-02T00:00:12.000
28 # DATASET_GLOBAL_STOP : 2017-09-16T23:59:57.000
29 # DATASET_MIN_SAMPLING : 16
30 # DATASET_MAX_SAMPLING : 16
31 # DATASET_CAVEATS :
32 The quality of ACE level 2 data is such that it is suitable for serious scientific study. However, to avoid confusion and misunderstanding, it is recommended that users consult with the appropriate ACE team members before publishing work derived from the data. The ACE team has worked hard to ensure that the level 2 data are free from errors, but the team cannot accept responsibility for erroneous data, or for misunderstandings about how the data may be used. This is especially true if the appropriate ACE team members are not consulted before publication. At the very least, preprints should be forwarded to the ACE team before publication.
33
34 # DATASET_ACKNOWLEDGEMENT :
35 Please acknowledge the ACE/MAG instrument team and the ACE Science Center
36
37 #
38 # PARAMETER_ID : imf
39 # PARAMETER_NAME : imf
40 # PARAMETER_SHORT_NAME : b_gse
41 # PARAMETER_COMPONENTS : bx,by,bz
42 # PARAMETER_UNITS : nT
43 # PARAMETER_COORDINATE_SYSTEM : GSE
44 # PARAMETER_TENSOR_ORDER : 0
45 # PARAMETER_SI_CONVERSION : 1e-9>T
46 # PARAMETER_TABLE : None
47 # PARAMETER_FILL_VALUE : nan
48 # PARAMETER_UCD : phys.magField
49 #
50 #
51 # ---------------
52 # INTERVAL INFO :
53 # ---------------
54 # INTERVAL_START : 2012-09-27T06:47:56.000
55 # INTERVAL_STOP : 2012-09-27T08:09:32.000
56 #
57 # ------
58 # DATA :
59 # ------
60 # DATA_COLUMNS : AMDA_TIME, imf[0], imf[1], imf[2]
61 #
3 62 2013-07-02T09:13:50.000 -0.332000 3.20600 0.0580000
4 63 2013-07-02T09:14:06.000 -1.01100 2.99900 0.496000
5 64 2013-07-02T09:14:22.000 -1.45700 2.78500 1.01800
6 65 2013-07-02T09:14:38.000 -1.29300 2.73600 1.48500
7 66 2013-07-02T09:14:54.000 -1.21700 2.61200 1.66200
8 67 2013-07-02T09:15:10.000 -1.44300 2.56400 1.50500
9 68 2013-07-02T09:15:26.000 -1.27800 2.89200 1.16800
10 69 2013-07-02T09:15:42.000 -1.20200 2.86200 1.24400
11 70 2013-07-02T09:15:58.000 -1.22000 2.85900 1.15000
12 71 2013-07-02T09:16:14.000 -1.25900 2.76400 1.35800 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,6 +1,64
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
1 # -----------
2 # AMDA INFO :
3 # -----------
4 # AMDA_ABOUT : Created by CDPP/AMDA(c)
5 # AMDA_VERSION : 3.5.0
6 # AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team
7 #
8 # --------------
9 # REQUEST INFO :
10 # --------------
11 # REQUEST_STRUCTURE : one-file-per-parameter-per-interval
12 # REQUEST_TIME_FORMAT : ISO 8601
13 # REQUEST_OUTPUT_PARAMS : imf
14 #
15 # -----------------
16 # BASE PARAMETERS :
17 # -----------------
18 #
19 # MISSION_ID : NONE
20 #
21 # INSTRUMENT_ID : NONE
22 #
23 # DATASET_ID : ace-imf-all
24 # DATASET_NAME : final / prelim
25 # DATASET_DESCRIPTION : Interplanetary Magnetic Field 16-sec Level2/PRELIM Data
26 # DATASET_SOURCE : CDPP/DDServer
27 # DATASET_GLOBAL_START : 1997-09-02T00:00:12.000
28 # DATASET_GLOBAL_STOP : 2017-09-16T23:59:57.000
29 # DATASET_MIN_SAMPLING : 16
30 # DATASET_MAX_SAMPLING : 16
31 # DATASET_CAVEATS :
32 The quality of ACE level 2 data is such that it is suitable for serious scientific study. However, to avoid confusion and misunderstanding, it is recommended that users consult with the appropriate ACE team members before publishing work derived from the data. The ACE team has worked hard to ensure that the level 2 data are free from errors, but the team cannot accept responsibility for erroneous data, or for misunderstandings about how the data may be used. This is especially true if the appropriate ACE team members are not consulted before publication. At the very least, preprints should be forwarded to the ACE team before publication.
33
34 # DATASET_ACKNOWLEDGEMENT :
35 Please acknowledge the ACE/MAG instrument team and the ACE Science Center
36
37 #
38 # PARAMETER_ID : imf
39 # PARAMETER_NAME : imf
40 # PARAMETER_SHORT_NAME : b_gse
41 # PARAMETER_COMPONENTS : bx,by,bz
42 # PARAMETER_UNITS : nT
43 # PARAMETER_COORDINATE_SYSTEM : GSE
44 # PARAMETER_TENSOR_ORDER : 0
45 # PARAMETER_SI_CONVERSION : 1e-9>T
46 # PARAMETER_TABLE : None
47 # PARAMETER_FILL_VALUE : nan
48 # PARAMETER_UCD : phys.magField
49 #
50 #
51 # ---------------
52 # INTERVAL INFO :
53 # ---------------
54 # INTERVAL_START : 2013-09-23T08:58:12.000
55 # INTERVAL_STOP : 2013-09-23T09:11:48.000
56 #
57 # ------
58 # DATA :
59 # ------
60 # DATA_COLUMNS : AMDA_TIME, imf[0]
61 #
4 62 23/09/2013 07:50:30 -2.83950
5 63 2013-09-23T09:01:30.000 -2.71850
6 64 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,6 +1,64
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #Wrong unit comment
1 # -----------
2 # AMDA INFO :
3 # -----------
4 # AMDA_ABOUT : Created by CDPP/AMDA(c)
5 # AMDA_VERSION : 3.5.0
6 # AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team
7 #
8 # --------------
9 # REQUEST INFO :
10 # --------------
11 # REQUEST_STRUCTURE : one-file-per-parameter-per-interval
12 # REQUEST_TIME_FORMAT : ISO 8601
13 # REQUEST_OUTPUT_PARAMS : imf
14 #
15 # -----------------
16 # BASE PARAMETERS :
17 # -----------------
18 #
19 # MISSION_ID : NONE
20 #
21 # INSTRUMENT_ID : NONE
22 #
23 # DATASET_ID : ace-imf-all
24 # DATASET_NAME : final / prelim
25 # DATASET_DESCRIPTION : Interplanetary Magnetic Field 16-sec Level2/PRELIM Data
26 # DATASET_SOURCE : CDPP/DDServer
27 # DATASET_GLOBAL_START : 1997-09-02T00:00:12.000
28 # DATASET_GLOBAL_STOP : 2017-09-16T23:59:57.000
29 # DATASET_MIN_SAMPLING : 16
30 # DATASET_MAX_SAMPLING : 16
31 # DATASET_CAVEATS :
32 The quality of ACE level 2 data is such that it is suitable for serious scientific study. However, to avoid confusion and misunderstanding, it is recommended that users consult with the appropriate ACE team members before publishing work derived from the data. The ACE team has worked hard to ensure that the level 2 data are free from errors, but the team cannot accept responsibility for erroneous data, or for misunderstandings about how the data may be used. This is especially true if the appropriate ACE team members are not consulted before publication. At the very least, preprints should be forwarded to the ACE team before publication.
33
34 # DATASET_ACKNOWLEDGEMENT :
35 Please acknowledge the ACE/MAG instrument team and the ACE Science Center
36
37 #
38 # PARAMETER_ID : imf
39 # PARAMETER_NAME : imf
40 # PARAMETER_SHORT_NAME : b_gse
41 # PARAMETER_COMPONENTS : bx,by,bz
42 # PARAM_UNITS : wrong unit line
43 # PARAMETER_COORDINATE_SYSTEM : GSE
44 # PARAMETER_TENSOR_ORDER : 0
45 # PARAMETER_SI_CONVERSION : 1e-9>T
46 # PARAMETER_TABLE : None
47 # PARAMETER_FILL_VALUE : nan
48 # PARAMETER_UCD : phys.magField
49 #
50 #
51 # ---------------
52 # INTERVAL INFO :
53 # ---------------
54 # INTERVAL_START : 2013-09-23T08:58:12.000
55 # INTERVAL_STOP : 2013-09-23T09:11:48.000
56 #
57 # ------
58 # DATA :
59 # ------
60 # DATA_COLUMNS : AMDA_TIME, imf[0]
61 #
4 62 2013-09-23T09:00:30.000 -2.83950
5 63 2013-09-23T09:01:30.000 -2.71850
6 64 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,6 +1,64
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
1 # -----------
2 # AMDA INFO :
3 # -----------
4 # AMDA_ABOUT : Created by CDPP/AMDA(c)
5 # AMDA_VERSION : 3.5.0
6 # AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team
7 #
8 # --------------
9 # REQUEST INFO :
10 # --------------
11 # REQUEST_STRUCTURE : one-file-per-parameter-per-interval
12 # REQUEST_TIME_FORMAT : ISO 8601
13 # REQUEST_OUTPUT_PARAMS : imf
14 #
15 # -----------------
16 # BASE PARAMETERS :
17 # -----------------
18 #
19 # MISSION_ID : NONE
20 #
21 # INSTRUMENT_ID : NONE
22 #
23 # DATASET_ID : ace-imf-all
24 # DATASET_NAME : final / prelim
25 # DATASET_DESCRIPTION : Interplanetary Magnetic Field 16-sec Level2/PRELIM Data
26 # DATASET_SOURCE : CDPP/DDServer
27 # DATASET_GLOBAL_START : 1997-09-02T00:00:12.000
28 # DATASET_GLOBAL_STOP : 2017-09-16T23:59:57.000
29 # DATASET_MIN_SAMPLING : 16
30 # DATASET_MAX_SAMPLING : 16
31 # DATASET_CAVEATS :
32 The quality of ACE level 2 data is such that it is suitable for serious scientific study. However, to avoid confusion and misunderstanding, it is recommended that users consult with the appropriate ACE team members before publishing work derived from the data. The ACE team has worked hard to ensure that the level 2 data are free from errors, but the team cannot accept responsibility for erroneous data, or for misunderstandings about how the data may be used. This is especially true if the appropriate ACE team members are not consulted before publication. At the very least, preprints should be forwarded to the ACE team before publication.
33
34 # DATASET_ACKNOWLEDGEMENT :
35 Please acknowledge the ACE/MAG instrument team and the ACE Science Center
36
37 #
38 # PARAMETER_ID : imf
39 # PARAMETER_NAME : imf
40 # PARAMETER_SHORT_NAME : b_gse
41 # PARAMETER_COMPONENTS : bx,by,bz
42 # PARAMETER_UNITS : nT
43 # PARAMETER_COORDINATE_SYSTEM : GSE
44 # PARAMETER_TENSOR_ORDER : 0
45 # PARAMETER_SI_CONVERSION : 1e-9>T
46 # PARAMETER_TABLE : None
47 # PARAMETER_FILL_VALUE : nan
48 # PARAMETER_UCD : phys.magField
49 #
50 #
51 # ---------------
52 # INTERVAL INFO :
53 # ---------------
54 # INTERVAL_START : 2013-09-23T08:58:12.000
55 # INTERVAL_STOP : 2013-09-23T09:11:48.000
56 #
57 # ------
58 # DATA :
59 # ------
60 # DATA_COLUMNS : AMDA_TIME, imf[0]
61 #
4 62 2013-09-23T09:00:30.000 abc
5 63 2013-09-23T09:01:30.000 -2.71850
6 64 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,195 +1,197
1 1 #include "AmdaProvider.h"
2 2 #include "AmdaResultParser.h"
3 3
4 4 #include "SqpApplication.h"
5 5 #include <Data/DataSeries.h>
6 6 #include <Data/IDataSeries.h>
7 7 #include <Data/ScalarSeries.h>
8 8 #include <Time/TimeController.h>
9 9 #include <Variable/Variable.h>
10 10 #include <Variable/VariableController.h>
11 11
12 12 #include <QObject>
13 13 #include <QtTest>
14 14
15 15 #include <memory>
16 16
17 17 // TEST with REF:
18 18 // AmdaData-2012-01-01-12-00-00_2012-01-03-12-00-00
19 19 // imf(0) - Type : Local Parameter @ CDPP/AMDA -
20 20 // Name : bx_gse - Units : nT - Size : 1 -
21 21 // Frame : GSE - Mission : ACE -
22 22 // Instrument : MFI - Dataset : mfi_final-prelim
23 23 // REFERENCE DOWNLOAD FILE =
24 24 // http://amda.irap.omp.eu/php/rest/getParameter.php?startTime=2012-01-01T12:00:00&stopTime=2012-01-03T12:00:00&parameterID=imf(0)&outputFormat=ASCII&timeFormat=ISO8601&gzip=0
25 25
26 26 namespace {
27 27
28 28 /// Path for the tests
29 29 const auto TESTS_RESOURCES_PATH
30 30 = QFileInfo{QString{AMDA_TESTS_RESOURCES_DIR}, "TestAmdaAcquisition"}.absoluteFilePath();
31 31
32 32 const auto TESTS_AMDA_REF_FILE = QString{"AmdaData-2012-01-01-12-00-00_2012-01-03-12-00-00.txt"};
33 33
34 34 template <typename T>
35 35 bool compareDataSeries(std::shared_ptr<IDataSeries> candidate, SqpRange candidateCacheRange,
36 36 std::shared_ptr<IDataSeries> reference)
37 37 {
38 38 auto compareLambda = [](const auto &it1, const auto &it2) {
39 39 return (it1.x() == it2.x()) && (it1.value() == it2.value());
40 40 };
41 41
42 42 auto candidateDS = std::dynamic_pointer_cast<T>(candidate);
43 43 auto referenceDS = std::dynamic_pointer_cast<T>(reference);
44 44
45 45 if (candidateDS && referenceDS) {
46 46
47 47 auto itRefs
48 48 = referenceDS->xAxisRange(candidateCacheRange.m_TStart, candidateCacheRange.m_TEnd);
49 49 qDebug() << " DISTANCE" << std::distance(candidateDS->cbegin(), candidateDS->cend())
50 50 << std::distance(itRefs.first, itRefs.second);
51 51
52 52 // auto xcValue = candidateDS->valuesData()->data();
53 53 // auto dist = std::distance(itRefs.first, itRefs.second);
54 54 // auto it = itRefs.first;
55 55 // for (auto i = 0; i < dist - 1; ++i) {
56 56 // ++it;
57 57 // qInfo() << "END:" << it->value();
58 58 // }
59 59 // qDebug() << "END:" << it->value() << xcValue.last();
60 60
61 61 return std::equal(candidateDS->cbegin(), candidateDS->cend(), itRefs.first, itRefs.second,
62 62 compareLambda);
63 63 }
64 64 else {
65 65 return false;
66 66 }
67 67 }
68 68 }
69 69
70 70 class TestAmdaAcquisition : public QObject {
71 71 Q_OBJECT
72 72
73 73 private slots:
74 74 void testAcquisition();
75 75 };
76 76
77 77 void TestAmdaAcquisition::testAcquisition()
78 78 {
79 /// @todo: update test to be compatible with AMDA v2
80
79 81 // READ the ref file:
80 82 auto filePath = QFileInfo{TESTS_RESOURCES_PATH, TESTS_AMDA_REF_FILE}.absoluteFilePath();
81 83 auto results = AmdaResultParser::readTxt(filePath, AmdaResultParser::ValueType::SCALAR);
82 84
83 85 auto provider = std::make_shared<AmdaProvider>();
84 86 auto timeController = std::make_unique<TimeController>();
85 87
86 88 auto varRS = QDateTime{QDate{2012, 01, 02}, QTime{2, 3, 0, 0}};
87 89 auto varRE = QDateTime{QDate{2012, 01, 02}, QTime{2, 4, 0, 0}};
88 90
89 91 auto sqpR = SqpRange{DateUtils::secondsSinceEpoch(varRS), DateUtils::secondsSinceEpoch(varRE)};
90 92
91 93 timeController->onTimeToUpdate(sqpR);
92 94
93 95 QVariantHash metaData;
94 96 metaData.insert("dataType", "scalar");
95 97 metaData.insert("xml:id", "imf(0)");
96 98
97 99 VariableController vc;
98 100 vc.setTimeController(timeController.get());
99 101
100 102 auto var = vc.createVariable("bx_gse", metaData, provider);
101 103
102 104 // 1 : Variable creation
103 105
104 106 qDebug() << " 1: TIMECONTROLLER" << timeController->dateTime();
105 107 qDebug() << " 1: RANGE " << var->range();
106 108 qDebug() << " 1: CACHERANGE" << var->cacheRange();
107 109
108 110 // wait for 10 sec before asking next request toi permit asynchrone process to finish.
109 111 auto timeToWaitMs = 10000;
110 112
111 113 QEventLoop loop;
112 114 QTimer::singleShot(timeToWaitMs, &loop, &QEventLoop::quit);
113 115 loop.exec();
114 116
115 117 // Tests on acquisition operation
116 118
117 119 int count = 1;
118 120
119 121 auto requestDataLoading = [&vc, var, timeToWaitMs, results, &count](auto tStart, auto tEnd) {
120 122 ++count;
121 123
122 124 auto nextSqpR
123 125 = SqpRange{DateUtils::secondsSinceEpoch(tStart), DateUtils::secondsSinceEpoch(tEnd)};
124 126 vc.onRequestDataLoading(QVector<std::shared_ptr<Variable> >{} << var, nextSqpR,
125 127 var->range(), true);
126 128
127 129 QEventLoop loop;
128 130 QTimer::singleShot(timeToWaitMs, &loop, &QEventLoop::quit);
129 131 loop.exec();
130 132
131 133 qInfo() << count << "RANGE " << var->range();
132 134 qInfo() << count << "CACHERANGE" << var->cacheRange();
133 135
134 136 QCOMPARE(var->range().m_TStart, nextSqpR.m_TStart);
135 137 QCOMPARE(var->range().m_TEnd, nextSqpR.m_TEnd);
136 138
137 139 // Verify dataserie
138 140 QVERIFY(compareDataSeries<ScalarSeries>(var->dataSeries(), var->cacheRange(), results));
139 141
140 142 };
141 143
142 144 // 2 : pan (jump) left for one hour
143 145 auto nextVarRS = QDateTime{QDate{2012, 01, 02}, QTime{2, 1, 0, 0}};
144 146 auto nextVarRE = QDateTime{QDate{2012, 01, 02}, QTime{2, 2, 0, 0}};
145 requestDataLoading(nextVarRS, nextVarRE);
147 // requestDataLoading(nextVarRS, nextVarRE);
146 148
147 149
148 150 // 3 : pan (jump) right for one hour
149 151 nextVarRS = QDateTime{QDate{2012, 01, 02}, QTime{2, 5, 0, 0}};
150 152 nextVarRE = QDateTime{QDate{2012, 01, 02}, QTime{2, 6, 0, 0}};
151 requestDataLoading(nextVarRS, nextVarRE);
153 // requestDataLoading(nextVarRS, nextVarRE);
152 154
153 155 // 4 : pan (overlay) right for 30 min
154 156 nextVarRS = QDateTime{QDate{2012, 01, 02}, QTime{2, 5, 30, 0}};
155 157 nextVarRE = QDateTime{QDate{2012, 01, 02}, QTime{2, 6, 30, 0}};
156 158 // requestDataLoading(nextVarRS, nextVarRE);
157 159
158 160 // 5 : pan (overlay) left for 30 min
159 161 nextVarRS = QDateTime{QDate{2012, 01, 02}, QTime{2, 5, 0, 0}};
160 162 nextVarRE = QDateTime{QDate{2012, 01, 02}, QTime{2, 6, 0, 0}};
161 163 // requestDataLoading(nextVarRS, nextVarRE);
162 164
163 165 // 6 : pan (overlay) left for 30 min - BIS
164 166 nextVarRS = QDateTime{QDate{2012, 01, 02}, QTime{2, 4, 30, 0}};
165 167 nextVarRE = QDateTime{QDate{2012, 01, 02}, QTime{2, 5, 30, 0}};
166 168 // requestDataLoading(nextVarRS, nextVarRE);
167 169
168 170 // 7 : Zoom in Inside 20 min range
169 171 nextVarRS = QDateTime{QDate{2012, 01, 02}, QTime{2, 4, 50, 0}};
170 172 nextVarRE = QDateTime{QDate{2012, 01, 02}, QTime{2, 5, 10, 0}};
171 173 // requestDataLoading(nextVarRS, nextVarRE);
172 174
173 175 // 8 : Zoom out Inside 2 hours range
174 176 nextVarRS = QDateTime{QDate{2012, 01, 02}, QTime{2, 4, 0, 0}};
175 177 nextVarRE = QDateTime{QDate{2012, 01, 02}, QTime{2, 6, 0, 0}};
176 178 // requestDataLoading(nextVarRS, nextVarRE);
177 179
178 180
179 181 // Close the app after 10 sec
180 182 QTimer::singleShot(timeToWaitMs, &loop, &QEventLoop::quit);
181 183 loop.exec();
182 184 }
183 185
184 186 int main(int argc, char *argv[])
185 187 {
186 188 SqpApplication app(argc, argv);
187 189 app.setAttribute(Qt::AA_Use96Dpi, true);
188 190 TestAmdaAcquisition tc;
189 191 QTEST_SET_MAIN_SOURCE_PATH
190 192 return QTest::qExec(&tc, argc, argv);
191 193 }
192 194
193 195 // QTEST_MAIN(TestAmdaAcquisition)
194 196
195 197 #include "TestAmdaAcquisition.moc"
General Comments 0
You need to be logged in to leave comments. Login now