##// END OF EJS Templates
GIT_REPOSITORIES » LPP » Users » mperrinel » SciQLop-fork
RSS

Fork of HG_REPOSITORIES/LPP/SciQLOP_Repos/SciQLop

Fixes AMDA crash
Alexandre Leroux -
r972:acd0e387536c
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,423 +1,423
1 1 #include "AmdaResultParserHelper.h"
2 2
3 3 #include <Common/DateUtils.h>
4 4 #include <Common/SortUtils.h>
5 5
6 6 #include <Data/DataSeriesUtils.h>
7 7 #include <Data/ScalarSeries.h>
8 8 #include <Data/SpectrogramSeries.h>
9 9 #include <Data/Unit.h>
10 10 #include <Data/VectorSeries.h>
11 11
12 12 #include <QtCore/QDateTime>
13 13 #include <QtCore/QRegularExpression>
14 14
15 15 #include <functional>
16 16
17 17 Q_LOGGING_CATEGORY(LOG_AmdaResultParserHelper, "AmdaResultParserHelper")
18 18
19 19 namespace {
20 20
21 21 // ///////// //
22 22 // Constants //
23 23 // ///////// //
24 24
25 25 /// Separator between values in a result line
26 26 const auto RESULT_LINE_SEPARATOR = QRegularExpression{QStringLiteral("\\s+")};
27 27
28 28 /// Format for dates in result files
29 29 const auto DATE_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss.zzz");
30 30
31 31 // /////// //
32 32 // Methods //
33 33 // /////// //
34 34
35 35 /**
36 36 * Checks that the properties contain a specific unit and that this unit is valid
37 37 * @param properties the properties map in which to search unit
38 38 * @param key the key to search for the unit in the properties
39 39 * @param errorMessage the error message to log in case the unit is invalid
40 40 * @return true if the unit is valid, false it it's invalid or was not found in the properties
41 41 */
42 42 bool checkUnit(const Properties &properties, const QString &key, const QString &errorMessage)
43 43 {
44 44 auto unit = properties.value(key).value<Unit>();
45 45 if (unit.m_Name.isEmpty()) {
46 46 qCWarning(LOG_AmdaResultParserHelper()) << errorMessage;
47 47 return false;
48 48 }
49 49
50 50 return true;
51 51 }
52 52
53 53 QDateTime dateTimeFromString(const QString &stringDate) noexcept
54 54 {
55 55 #if QT_VERSION >= QT_VERSION_CHECK(5, 8, 0)
56 56 return QDateTime::fromString(stringDate, Qt::ISODateWithMs);
57 57 #else
58 58 return QDateTime::fromString(stringDate, DATE_FORMAT);
59 59 #endif
60 60 }
61 61
62 62 /// Converts a string date to a double date
63 63 /// @return a double that represents the date in seconds, NaN if the string date can't be converted
64 64 double doubleDate(const QString &stringDate) noexcept
65 65 {
66 66 // Format: yyyy-MM-ddThh:mm:ss.zzz
67 67 auto dateTime = dateTimeFromString(stringDate);
68 68 dateTime.setTimeSpec(Qt::UTC);
69 69 return dateTime.isValid() ? DateUtils::secondsSinceEpoch(dateTime)
70 70 : std::numeric_limits<double>::quiet_NaN();
71 71 }
72 72
73 73 /**
74 74 * Reads a line from the AMDA file and tries to extract a x-axis data and value data from it
75 75 * @param xAxisData the vector in which to store the x-axis data extracted
76 76 * @param valuesData the vector in which to store the value extracted
77 77 * @param line the line to read to extract the property
78 78 * @param valuesIndexes indexes of insertion of read values. For example, if the line contains three
79 79 * columns of values, and valuesIndexes are {2, 0, 1}, the value of the third column will be read
80 80 * and inserted first, then the value of the first column, and finally the value of the second
81 81 * column.
82 82 * @param fillValue value that tags an invalid data. For example, if fillValue is -1 and a read
83 83 * value is -1, then this value is considered as invalid and converted to NaN
84 84 */
85 85 void tryReadResult(std::vector<double> &xAxisData, std::vector<double> &valuesData,
86 86 const QString &line, const std::vector<int> &valuesIndexes,
87 87 double fillValue = std::numeric_limits<double>::quiet_NaN())
88 88 {
89 89 auto lineData = line.split(RESULT_LINE_SEPARATOR, QString::SkipEmptyParts);
90 90
91 91 // Checks that the line contains expected number of values + x-axis value
92 92 if (lineData.size() == valuesIndexes.size() + 1) {
93 93 // X : the data is converted from date to double (in secs)
94 94 auto x = doubleDate(lineData.at(0));
95 95
96 96 // Adds result only if x is valid. Then, if value is invalid, it is set to NaN
97 97 if (!std::isnan(x)) {
98 98 xAxisData.push_back(x);
99 99
100 100 // Values
101 101 for (auto valueIndex : valuesIndexes) {
102 102 bool valueOk;
103 103 // we use valueIndex + 1 to skip column 0 (x-axis value)
104 104 auto value = lineData.at(valueIndex + 1).toDouble(&valueOk);
105 105
106 106 if (!valueOk) {
107 107 qCWarning(LOG_AmdaResultParserHelper())
108 108 << QObject::tr(
109 109 "Value from (line %1, column %2) is invalid and will be "
110 110 "converted to NaN")
111 111 .arg(line, valueIndex);
112 112 value = std::numeric_limits<double>::quiet_NaN();
113 113 }
114 114
115 115 // Handles fill value
116 116 if (!std::isnan(fillValue) && !std::isnan(value) && fillValue == value) {
117 117 value = std::numeric_limits<double>::quiet_NaN();
118 118 }
119 119
120 120 valuesData.push_back(value);
121 121 }
122 122 }
123 123 else {
124 124 qCWarning(LOG_AmdaResultParserHelper())
125 125 << QObject::tr("Can't retrieve results from line %1: x is invalid").arg(line);
126 126 }
127 127 }
128 128 else {
129 129 qCWarning(LOG_AmdaResultParserHelper())
130 130 << QObject::tr("Can't retrieve results from line %1: invalid line").arg(line);
131 131 }
132 132 }
133 133
134 134 /**
135 135 * Reads a line from the AMDA file and tries to extract a property from it
136 136 * @param properties the properties map in which to put the property extracted from the line
137 137 * @param key the key to which the property is added in the properties map
138 138 * @param line the line to read to extract the property
139 139 * @param regex the expected regex to extract the property. If the line matches this regex, the
140 140 * property is generated
141 141 * @param fun the function used to generate the property
142 142 * @return true if the property could be generated, false if the line does not match the regex, or
143 143 * if a property has already been generated for the key
144 144 */
145 145 template <typename GeneratePropertyFun>
146 146 bool tryReadProperty(Properties &properties, const QString &key, const QString &line,
147 147 const QRegularExpression &regex, GeneratePropertyFun fun)
148 148 {
149 149 if (properties.contains(key)) {
150 150 return false;
151 151 }
152 152
153 153 auto match = regex.match(line);
154 154 if (match.hasMatch()) {
155 155 properties.insert(key, fun(match));
156 156 }
157 157
158 158 return match.hasMatch();
159 159 }
160 160
161 161 /**
162 162 * Reads a line from the AMDA file and tries to extract a data from it. Date is converted to double
163 163 * @sa tryReadProperty()
164 164 */
165 165 bool tryReadDate(Properties &properties, const QString &key, const QString &line,
166 166 const QRegularExpression &regex, bool timeUnit = false)
167 167 {
168 168 return tryReadProperty(properties, key, line, regex, [timeUnit](const auto &match) {
169 169 return QVariant::fromValue(doubleDate(match.captured(1)));
170 170 });
171 171 }
172 172
173 173 /**
174 174 * Reads a line from the AMDA file and tries to extract a double from it
175 175 * @sa tryReadProperty()
176 176 */
177 177 bool tryReadDouble(Properties &properties, const QString &key, const QString &line,
178 178 const QRegularExpression &regex)
179 179 {
180 180 return tryReadProperty(properties, key, line, regex, [](const auto &match) {
181 181 bool ok;
182 182
183 183 // If the value can't be converted to double, it is set to NaN
184 184 auto doubleValue = match.captured(1).toDouble(&ok);
185 185 if (!ok) {
186 186 doubleValue = std::numeric_limits<double>::quiet_NaN();
187 187 }
188 188
189 189 return QVariant::fromValue(doubleValue);
190 190 });
191 191 }
192 192
193 193 /**
194 194 * Reads a line from the AMDA file and tries to extract a vector of doubles from it
195 195 * @param sep the separator of double values in the line
196 196 * @sa tryReadProperty()
197 197 */
198 198 bool tryReadDoubles(Properties &properties, const QString &key, const QString &line,
199 199 const QRegularExpression &regex, const QString &sep = QStringLiteral(","))
200 200 {
201 201 return tryReadProperty(properties, key, line, regex, [sep](const auto &match) {
202 202 std::vector<double> doubleValues{};
203 203
204 204 // If the value can't be converted to double, it is set to NaN
205 205 auto values = match.captured(1).split(sep);
206 206 for (auto value : values) {
207 207 bool ok;
208 208
209 209 auto doubleValue = value.toDouble(&ok);
210 210 if (!ok) {
211 211 doubleValue = std::numeric_limits<double>::quiet_NaN();
212 212 }
213 213
214 214 doubleValues.push_back(doubleValue);
215 215 }
216 216
217 217 return QVariant::fromValue(doubleValues);
218 218 });
219 219 }
220 220
221 221 /**
222 222 * Reads a line from the AMDA file and tries to extract a unit from it
223 223 * @sa tryReadProperty()
224 224 */
225 225 bool tryReadUnit(Properties &properties, const QString &key, const QString &line,
226 226 const QRegularExpression &regex, bool timeUnit = false)
227 227 {
228 228 return tryReadProperty(properties, key, line, regex, [timeUnit](const auto &match) {
229 229 return QVariant::fromValue(Unit{match.captured(1), timeUnit});
230 230 });
231 231 }
232 232
233 233 } // namespace
234 234
235 235 // ////////////////// //
236 236 // ScalarParserHelper //
237 237 // ////////////////// //
238 238
239 239 bool ScalarParserHelper::checkProperties()
240 240 {
241 241 return checkUnit(m_Properties, X_AXIS_UNIT_PROPERTY,
242 242 QObject::tr("The x-axis unit could not be found in the file"));
243 243 }
244 244
245 245 std::shared_ptr<IDataSeries> ScalarParserHelper::createSeries()
246 246 {
247 247 return std::make_shared<ScalarSeries>(std::move(m_XAxisData), std::move(m_ValuesData),
248 248 m_Properties.value(X_AXIS_UNIT_PROPERTY).value<Unit>(),
249 249 m_Properties.value(VALUES_UNIT_PROPERTY).value<Unit>());
250 250 }
251 251
252 252 void ScalarParserHelper::readPropertyLine(const QString &line)
253 253 {
254 254 tryReadUnit(m_Properties, X_AXIS_UNIT_PROPERTY, line, DEFAULT_X_AXIS_UNIT_REGEX, true);
255 255 }
256 256
257 257 void ScalarParserHelper::readResultLine(const QString &line)
258 258 {
259 259 tryReadResult(m_XAxisData, m_ValuesData, line, valuesIndexes());
260 260 }
261 261
262 262 std::vector<int> ScalarParserHelper::valuesIndexes() const
263 263 {
264 264 // Only one value to read
265 265 static auto result = std::vector<int>{0};
266 266 return result;
267 267 }
268 268
269 269 // /////////////////////// //
270 270 // SpectrogramParserHelper //
271 271 // /////////////////////// //
272 272
273 273 bool SpectrogramParserHelper::checkProperties()
274 274 {
275 275 // Generates y-axis data from bands extracted (take the middle of the intervals)
276 276 auto minBands = m_Properties.value(MIN_BANDS_PROPERTY).value<std::vector<double> >();
277 277 auto maxBands = m_Properties.value(MAX_BANDS_PROPERTY).value<std::vector<double> >();
278 278
279 if (minBands.size() != maxBands.size()) {
279 if (minBands.size() < 2 || minBands.size() != maxBands.size()) {
280 280 qCWarning(LOG_AmdaResultParserHelper()) << QObject::tr(
281 281 "Can't generate y-axis data from bands extracted: bands intervals are invalid");
282 282 return false;
283 283 }
284 284
285 285 std::transform(
286 286 minBands.begin(), minBands.end(), maxBands.begin(), std::back_inserter(m_YAxisData),
287 287 [](const auto &minValue, const auto &maxValue) { return (minValue + maxValue) / 2.; });
288 288
289 289 // Generates values indexes, i.e. the order in which each value will be retrieved (in ascending
290 290 // order of the associated bands)
291 291 m_ValuesIndexes = SortUtils::sortPermutation(m_YAxisData, std::less<double>());
292 292
293 293 // Sorts y-axis data accoding to the ascending order
294 294 m_YAxisData = SortUtils::sort(m_YAxisData, 1, m_ValuesIndexes);
295 295
296 296 // Sets fill value
297 297 m_FillValue = m_Properties.value(FILL_VALUE_PROPERTY).value<double>();
298 298
299 299 return true;
300 300 }
301 301
302 302 std::shared_ptr<IDataSeries> SpectrogramParserHelper::createSeries()
303 303 {
304 304 // Before creating the series, we handle its data holes
305 305 handleDataHoles();
306 306
307 307 return std::make_shared<SpectrogramSeries>(
308 308 std::move(m_XAxisData), std::move(m_YAxisData), std::move(m_ValuesData),
309 309 Unit{"t", true}, // x-axis unit is always a time unit
310 310 m_Properties.value(Y_AXIS_UNIT_PROPERTY).value<Unit>(),
311 311 m_Properties.value(VALUES_UNIT_PROPERTY).value<Unit>(),
312 312 m_Properties.value(MIN_SAMPLING_PROPERTY).value<double>());
313 313 }
314 314
315 315 void SpectrogramParserHelper::readPropertyLine(const QString &line)
316 316 {
317 317 // Set of functions to test on the line to generate a property. If a function is valid (i.e. a
318 318 // property has been generated for the line), the line is treated as processed and the other
319 319 // functions are not called
320 320 std::vector<std::function<bool()> > functions{
321 321 // values unit
322 322 [&] {
323 323 return tryReadUnit(m_Properties, VALUES_UNIT_PROPERTY, line,
324 324 SPECTROGRAM_VALUES_UNIT_REGEX);
325 325 },
326 326 // y-axis unit
327 327 [&] {
328 328 return tryReadUnit(m_Properties, Y_AXIS_UNIT_PROPERTY, line,
329 329 SPECTROGRAM_Y_AXIS_UNIT_REGEX);
330 330 },
331 331 // min sampling
332 332 [&] {
333 333 return tryReadDouble(m_Properties, MIN_SAMPLING_PROPERTY, line,
334 334 SPECTROGRAM_MIN_SAMPLING_REGEX);
335 335 },
336 336 // max sampling
337 337 [&] {
338 338 return tryReadDouble(m_Properties, MAX_SAMPLING_PROPERTY, line,
339 339 SPECTROGRAM_MAX_SAMPLING_REGEX);
340 340 },
341 341 // fill value
342 342 [&] {
343 343 return tryReadDouble(m_Properties, FILL_VALUE_PROPERTY, line,
344 344 SPECTROGRAM_FILL_VALUE_REGEX);
345 345 },
346 346 // min bounds of each band
347 347 [&] {
348 348 return tryReadDoubles(m_Properties, MIN_BANDS_PROPERTY, line,
349 349 SPECTROGRAM_MIN_BANDS_REGEX);
350 350 },
351 351 // max bounds of each band
352 352 [&] {
353 353 return tryReadDoubles(m_Properties, MAX_BANDS_PROPERTY, line,
354 354 SPECTROGRAM_MAX_BANDS_REGEX);
355 355 },
356 356 // start time of data
357 357 [&] {
358 358 return tryReadDate(m_Properties, START_TIME_PROPERTY, line,
359 359 SPECTROGRAM_START_TIME_REGEX);
360 360 },
361 361 // end time of data
362 362 [&] {
363 363 return tryReadDate(m_Properties, END_TIME_PROPERTY, line, SPECTROGRAM_END_TIME_REGEX);
364 364 }};
365 365
366 366 for (auto function : functions) {
367 367 // Stops at the first function that is valid
368 368 if (function()) {
369 369 return;
370 370 }
371 371 }
372 372 }
373 373
374 374 void SpectrogramParserHelper::readResultLine(const QString &line)
375 375 {
376 376 tryReadResult(m_XAxisData, m_ValuesData, line, m_ValuesIndexes, m_FillValue);
377 377 }
378 378
379 379 void SpectrogramParserHelper::handleDataHoles()
380 380 {
381 381 // Fills data holes according to the max resolution found in the AMDA file
382 382 auto resolution = m_Properties.value(MAX_SAMPLING_PROPERTY).value<double>();
383 383 auto fillValue = m_Properties.value(FILL_VALUE_PROPERTY).value<double>();
384 384 auto minBound = m_Properties.value(START_TIME_PROPERTY).value<double>();
385 385 auto maxBound = m_Properties.value(END_TIME_PROPERTY).value<double>();
386 386
387 387 DataSeriesUtils::fillDataHoles(m_XAxisData, m_ValuesData, resolution, fillValue, minBound,
388 388 maxBound);
389 389 }
390 390
391 391 // ////////////////// //
392 392 // VectorParserHelper //
393 393 // ////////////////// //
394 394
395 395 bool VectorParserHelper::checkProperties()
396 396 {
397 397 return checkUnit(m_Properties, X_AXIS_UNIT_PROPERTY,
398 398 QObject::tr("The x-axis unit could not be found in the file"));
399 399 }
400 400
401 401 std::shared_ptr<IDataSeries> VectorParserHelper::createSeries()
402 402 {
403 403 return std::make_shared<VectorSeries>(std::move(m_XAxisData), std::move(m_ValuesData),
404 404 m_Properties.value(X_AXIS_UNIT_PROPERTY).value<Unit>(),
405 405 m_Properties.value(VALUES_UNIT_PROPERTY).value<Unit>());
406 406 }
407 407
408 408 void VectorParserHelper::readPropertyLine(const QString &line)
409 409 {
410 410 tryReadUnit(m_Properties, X_AXIS_UNIT_PROPERTY, line, DEFAULT_X_AXIS_UNIT_REGEX, true);
411 411 }
412 412
413 413 void VectorParserHelper::readResultLine(const QString &line)
414 414 {
415 415 tryReadResult(m_XAxisData, m_ValuesData, line, valuesIndexes());
416 416 }
417 417
418 418 std::vector<int> VectorParserHelper::valuesIndexes() const
419 419 {
420 420 // 3 values to read, in order in the file (x, y, z)
421 421 static auto result = std::vector<int>{0, 1, 2};
422 422 return result;
423 423 }
General Comments 0
You need to be logged in to leave comments. Login now