##// END OF EJS Templates
GIT_REPOSITORIES » LPP » Users » mperrinel » SciQLop-fork
RSS

Fork of HG_REPOSITORIES/LPP/SciQLOP_Repos/SciQLop

Handles bounds in spectrogram parser (1)...
Alexandre Leroux -
r1026:0b7501dc8683
parent child
Show More
Side by Side Unified
Context file:
r1026:0b7501dc8683 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,68 +1,70
1 1 #ifndef SCIQLOP_AMDARESULTPARSERDEFS_H
2 2 #define SCIQLOP_AMDARESULTPARSERDEFS_H
3 3
4 4 #include <QtCore/QRegularExpression>
5 5 #include <QtCore/QString>
6 6 #include <QtCore/QVariantHash>
7 7
8 8 // ////////// //
9 9 // Properties //
10 10 // ////////// //
11 11
12 12 /// Alias to represent properties read in the header of AMDA file
13 13 using Properties = QVariantHash;
14 14
15 extern const QString END_TIME_PROPERTY;
15 16 extern const QString FILL_VALUE_PROPERTY;
16 17 extern const QString MAX_BANDS_PROPERTY;
17 18 extern const QString MIN_BANDS_PROPERTY;
18 19 extern const QString MAX_SAMPLING_PROPERTY;
19 20 extern const QString MIN_SAMPLING_PROPERTY;
21 extern const QString START_TIME_PROPERTY;
20 22 extern const QString X_AXIS_UNIT_PROPERTY;
21 23 extern const QString Y_AXIS_UNIT_PROPERTY;
22 24 extern const QString VALUES_UNIT_PROPERTY;
23 25
24 26 // /////////////////// //
25 27 // Regular expressions //
26 28 // /////////////////// //
27 29
28 30 // AMDA V2
29 31 // /// Regex to find the header of the data in the file. This header indicates the end of comments
30 32 // in the file
31 33 // const auto DATA_HEADER_REGEX = QRegularExpression{QStringLiteral("#\\s*DATA\\s*:")};
32 34
33 35 // AMDA V2
34 36 // /// ... PARAMETER_UNITS : nT ...
35 37 // /// ... PARAMETER_UNITS:nT ...
36 38 // /// ... PARAMETER_UNITS: mΒ² ...
37 39 // /// ... PARAMETER_UNITS : m/s ...
38 40 // const auto UNIT_REGEX = QRegularExpression{QStringLiteral("\\s*PARAMETER_UNITS\\s*:\\s*(.+)")};
39 41
40 42 /// Regex to find x-axis unit in a line. Examples of valid lines:
41 43 /// ... - Units : nT - ...
42 44 /// ... -Units:nT- ...
43 45 /// ... -Units: mΒ²- ...
44 46 /// ... - Units : m/s - ...
45 47 extern const QRegularExpression DEFAULT_X_AXIS_UNIT_REGEX;
46 48
47 49 /// Regex to find fill value used in a line for a spectrogram
48 50 extern const QRegularExpression SPECTROGRAM_FILL_VALUE_REGEX;
49 51
50 52 /// Regex to find max bands in a line for a spectrogram
51 53 extern const QRegularExpression SPECTROGRAM_MAX_BANDS_REGEX;
52 54
53 55 /// Regex to find min bands in a line for a spectrogram
54 56 extern const QRegularExpression SPECTROGRAM_MIN_BANDS_REGEX;
55 57
56 58 /// Regex to find max x-axis sampling in a line for a spectrogram
57 59 extern const QRegularExpression SPECTROGRAM_MAX_SAMPLING_REGEX;
58 60
59 61 /// Regex to find min x-axis sampling in a line for a spectrogram
60 62 extern const QRegularExpression SPECTROGRAM_MIN_SAMPLING_REGEX;
61 63
62 64 /// Regex to find y-axis unit in a line for a spectrogram
63 65 extern const QRegularExpression SPECTROGRAM_Y_AXIS_UNIT_REGEX;
64 66
65 67 /// Regex to find values unit in a line for a spectrogram
66 68 extern const QRegularExpression SPECTROGRAM_VALUES_UNIT_REGEX;
67 69
68 70 #endif // SCIQLOP_AMDARESULTPARSERDEFS_H
Show file before | Show file after | Hide comments
@@ -1,34 +1,36
1 1 #include "AmdaResultParserDefs.h"
2 2
3 const QString END_TIME_PROPERTY = QStringLiteral("endTime");
3 4 const QString FILL_VALUE_PROPERTY = QStringLiteral("fillValue");
4 5 const QString MAX_BANDS_PROPERTY = QStringLiteral("maxBands");
5 6 const QString MIN_BANDS_PROPERTY = QStringLiteral("minBands");
6 7 const QString MAX_SAMPLING_PROPERTY = QStringLiteral("maxSampling");
7 8 const QString MIN_SAMPLING_PROPERTY = QStringLiteral("minSampling");
9 const QString START_TIME_PROPERTY = QStringLiteral("startTime");
8 10 const QString X_AXIS_UNIT_PROPERTY = QStringLiteral("xAxisUnit");
9 11 const QString Y_AXIS_UNIT_PROPERTY = QStringLiteral("yAxisUnit");
10 12 const QString VALUES_UNIT_PROPERTY = QStringLiteral("valuesUnit");
11 13
12 14 const QRegularExpression DEFAULT_X_AXIS_UNIT_REGEX
13 15 = QRegularExpression{QStringLiteral("-\\s*Units\\s*:\\s*(.+?)\\s*-")};
14 16
15 17 const QRegularExpression SPECTROGRAM_FILL_VALUE_REGEX
16 18 = QRegularExpression{QStringLiteral("\\s*PARAMETER_FILL_VALUE\\s*:\\s*(.*)")};
17 19
18 20 const QRegularExpression SPECTROGRAM_MAX_BANDS_REGEX
19 21 = QRegularExpression{QStringLiteral("\\s*PARAMETER_TABLE_MAX_VALUES\\[0\\]\\s*:\\s*(.*)")};
20 22
21 23 const QRegularExpression SPECTROGRAM_MIN_BANDS_REGEX
22 24 = QRegularExpression{QStringLiteral("\\s*PARAMETER_TABLE_MIN_VALUES\\[0\\]\\s*:\\s*(.*)")};
23 25
24 26 const QRegularExpression SPECTROGRAM_MAX_SAMPLING_REGEX
25 27 = QRegularExpression{QStringLiteral("\\s*DATASET_MAX_SAMPLING\\s*:\\s*(.*)")};
26 28
27 29 const QRegularExpression SPECTROGRAM_MIN_SAMPLING_REGEX
28 30 = QRegularExpression{QStringLiteral("\\s*DATASET_MIN_SAMPLING\\s*:\\s*(.*)")};
29 31
30 32 const QRegularExpression SPECTROGRAM_Y_AXIS_UNIT_REGEX
31 33 = QRegularExpression{QStringLiteral("\\s*PARAMETER_TABLE_UNITS\\[0\\]\\s*:\\s*(.*)")};
32 34
33 35 const QRegularExpression SPECTROGRAM_VALUES_UNIT_REGEX
34 36 = QRegularExpression{QStringLiteral("\\s*PARAMETER_UNITS\\s*:\\s*(.*)")};
Show file before | Show file after | Hide comments
@@ -1,398 +1,401
1 1 #include "AmdaResultParserHelper.h"
2 2
3 3 #include <Common/DateUtils.h>
4 4 #include <Common/SortUtils.h>
5 5
6 6 #include <Data/DataSeriesUtils.h>
7 7 #include <Data/ScalarSeries.h>
8 8 #include <Data/SpectrogramSeries.h>
9 9 #include <Data/Unit.h>
10 10 #include <Data/VectorSeries.h>
11 11
12 12 #include <QtCore/QDateTime>
13 13 #include <QtCore/QRegularExpression>
14 14
15 15 #include <functional>
16 16
17 17 Q_LOGGING_CATEGORY(LOG_AmdaResultParserHelper, "AmdaResultParserHelper")
18 18
19 19 namespace {
20 20
21 21 // ///////// //
22 22 // Constants //
23 23 // ///////// //
24 24
25 25 /// Separator between values in a result line
26 26 const auto RESULT_LINE_SEPARATOR = QRegularExpression{QStringLiteral("\\s+")};
27 27
28 28 /// Format for dates in result files
29 29 const auto DATE_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss.zzz");
30 30
31 31 // /////// //
32 32 // Methods //
33 33 // /////// //
34 34
35 35 /**
36 36 * Checks that the properties contain a specific unit and that this unit is valid
37 37 * @param properties the properties map in which to search unit
38 38 * @param key the key to search for the unit in the properties
39 39 * @param errorMessage the error message to log in case the unit is invalid
40 40 * @return true if the unit is valid, false it it's invalid or was not found in the properties
41 41 */
42 42 bool checkUnit(const Properties &properties, const QString &key, const QString &errorMessage)
43 43 {
44 44 auto unit = properties.value(key).value<Unit>();
45 45 if (unit.m_Name.isEmpty()) {
46 46 qCWarning(LOG_AmdaResultParserHelper()) << errorMessage;
47 47 return false;
48 48 }
49 49
50 50 return true;
51 51 }
52 52
53 53 QDateTime dateTimeFromString(const QString &stringDate) noexcept
54 54 {
55 55 #if QT_VERSION >= QT_VERSION_CHECK(5, 8, 0)
56 56 return QDateTime::fromString(stringDate, Qt::ISODateWithMs);
57 57 #else
58 58 return QDateTime::fromString(stringDate, DATE_FORMAT);
59 59 #endif
60 60 }
61 61
62 62 /// Converts a string date to a double date
63 63 /// @return a double that represents the date in seconds, NaN if the string date can't be converted
64 64 double doubleDate(const QString &stringDate) noexcept
65 65 {
66 66 // Format: yyyy-MM-ddThh:mm:ss.zzz
67 67 auto dateTime = dateTimeFromString(stringDate);
68 68 dateTime.setTimeSpec(Qt::UTC);
69 69 return dateTime.isValid() ? DateUtils::secondsSinceEpoch(dateTime)
70 70 : std::numeric_limits<double>::quiet_NaN();
71 71 }
72 72
73 73 /**
74 74 * Reads a line from the AMDA file and tries to extract a x-axis data and value data from it
75 75 * @param xAxisData the vector in which to store the x-axis data extracted
76 76 * @param valuesData the vector in which to store the value extracted
77 77 * @param line the line to read to extract the property
78 78 * @param valuesIndexes indexes of insertion of read values. For example, if the line contains three
79 79 * columns of values, and valuesIndexes are {2, 0, 1}, the value of the third column will be read
80 80 * and inserted first, then the value of the first column, and finally the value of the second
81 81 * column.
82 82 * @param fillValue value that tags an invalid data. For example, if fillValue is -1 and a read
83 83 * value is -1, then this value is considered as invalid and converted to NaN
84 84 */
85 85 void tryReadResult(std::vector<double> &xAxisData, std::vector<double> &valuesData,
86 86 const QString &line, const std::vector<int> &valuesIndexes,
87 87 double fillValue = std::numeric_limits<double>::quiet_NaN())
88 88 {
89 89 auto lineData = line.split(RESULT_LINE_SEPARATOR, QString::SkipEmptyParts);
90 90
91 91 // Checks that the line contains expected number of values + x-axis value
92 92 if (static_cast<size_t>(lineData.size()) == valuesIndexes.size() + 1) {
93 93 // X : the data is converted from date to double (in secs)
94 94 auto x = doubleDate(lineData.at(0));
95 95
96 96 // Adds result only if x is valid. Then, if value is invalid, it is set to NaN
97 97 if (!std::isnan(x)) {
98 98 xAxisData.push_back(x);
99 99
100 100 // Values
101 101 for (auto valueIndex : valuesIndexes) {
102 102 bool valueOk;
103 103 // we use valueIndex + 1 to skip column 0 (x-axis value)
104 104 auto value = lineData.at(valueIndex + 1).toDouble(&valueOk);
105 105
106 106 if (!valueOk) {
107 107 qCWarning(LOG_AmdaResultParserHelper())
108 108 << QObject::tr(
109 109 "Value from (line %1, column %2) is invalid and will be "
110 110 "converted to NaN")
111 111 .arg(line, valueIndex);
112 112 value = std::numeric_limits<double>::quiet_NaN();
113 113 }
114 114
115 115 // Handles fill value
116 116 if (!std::isnan(fillValue) && !std::isnan(value) && fillValue == value) {
117 117 value = std::numeric_limits<double>::quiet_NaN();
118 118 }
119 119
120 120 valuesData.push_back(value);
121 121 }
122 122 }
123 123 else {
124 124 qCWarning(LOG_AmdaResultParserHelper())
125 125 << QObject::tr("Can't retrieve results from line %1: x is invalid").arg(line);
126 126 }
127 127 }
128 128 else {
129 129 qCWarning(LOG_AmdaResultParserHelper())
130 130 << QObject::tr("Can't retrieve results from line %1: invalid line").arg(line);
131 131 }
132 132 }
133 133
134 134 /**
135 135 * Reads a line from the AMDA file and tries to extract a property from it
136 136 * @param properties the properties map in which to put the property extracted from the line
137 137 * @param key the key to which the property is added in the properties map
138 138 * @param line the line to read to extract the property
139 139 * @param regex the expected regex to extract the property. If the line matches this regex, the
140 140 * property is generated
141 141 * @param fun the function used to generate the property
142 142 * @return true if the property could be generated, false if the line does not match the regex, or
143 143 * if a property has already been generated for the key
144 144 */
145 145 template <typename GeneratePropertyFun>
146 146 bool tryReadProperty(Properties &properties, const QString &key, const QString &line,
147 147 const QRegularExpression &regex, GeneratePropertyFun fun)
148 148 {
149 149 if (properties.contains(key)) {
150 150 return false;
151 151 }
152 152
153 153 auto match = regex.match(line);
154 154 if (match.hasMatch()) {
155 155 properties.insert(key, fun(match));
156 156 }
157 157
158 158 return match.hasMatch();
159 159 }
160 160
161 161 /**
162 162 * Reads a line from the AMDA file and tries to extract a double from it
163 163 * @sa tryReadProperty()
164 164 */
165 165 bool tryReadDouble(Properties &properties, const QString &key, const QString &line,
166 166 const QRegularExpression &regex)
167 167 {
168 168 return tryReadProperty(properties, key, line, regex, [](const auto &match) {
169 169 bool ok;
170 170
171 171 // If the value can't be converted to double, it is set to NaN
172 172 auto doubleValue = match.captured(1).toDouble(&ok);
173 173 if (!ok) {
174 174 doubleValue = std::numeric_limits<double>::quiet_NaN();
175 175 }
176 176
177 177 return QVariant::fromValue(doubleValue);
178 178 });
179 179 }
180 180
181 181 /**
182 182 * Reads a line from the AMDA file and tries to extract a vector of doubles from it
183 183 * @param sep the separator of double values in the line
184 184 * @sa tryReadProperty()
185 185 */
186 186 bool tryReadDoubles(Properties &properties, const QString &key, const QString &line,
187 187 const QRegularExpression &regex, const QString &sep = QStringLiteral(","))
188 188 {
189 189 return tryReadProperty(properties, key, line, regex, [sep](const auto &match) {
190 190 std::vector<double> doubleValues{};
191 191
192 192 // If the value can't be converted to double, it is set to NaN
193 193 auto values = match.captured(1).split(sep);
194 194 for (auto value : values) {
195 195 bool ok;
196 196
197 197 auto doubleValue = value.toDouble(&ok);
198 198 if (!ok) {
199 199 doubleValue = std::numeric_limits<double>::quiet_NaN();
200 200 }
201 201
202 202 doubleValues.push_back(doubleValue);
203 203 }
204 204
205 205 return QVariant::fromValue(doubleValues);
206 206 });
207 207 }
208 208
209 209 /**
210 210 * Reads a line from the AMDA file and tries to extract a unit from it
211 211 * @sa tryReadProperty()
212 212 */
213 213 bool tryReadUnit(Properties &properties, const QString &key, const QString &line,
214 214 const QRegularExpression &regex, bool timeUnit = false)
215 215 {
216 216 return tryReadProperty(properties, key, line, regex, [timeUnit](const auto &match) {
217 217 return QVariant::fromValue(Unit{match.captured(1), timeUnit});
218 218 });
219 219 }
220 220
221 221 } // namespace
222 222
223 223 // ////////////////// //
224 224 // ScalarParserHelper //
225 225 // ////////////////// //
226 226
227 227 bool ScalarParserHelper::checkProperties()
228 228 {
229 229 return checkUnit(m_Properties, X_AXIS_UNIT_PROPERTY,
230 230 QObject::tr("The x-axis unit could not be found in the file"));
231 231 }
232 232
233 233 std::shared_ptr<IDataSeries> ScalarParserHelper::createSeries()
234 234 {
235 235 return std::make_shared<ScalarSeries>(std::move(m_XAxisData), std::move(m_ValuesData),
236 236 m_Properties.value(X_AXIS_UNIT_PROPERTY).value<Unit>(),
237 237 m_Properties.value(VALUES_UNIT_PROPERTY).value<Unit>());
238 238 }
239 239
240 240 void ScalarParserHelper::readPropertyLine(const QString &line)
241 241 {
242 242 tryReadUnit(m_Properties, X_AXIS_UNIT_PROPERTY, line, DEFAULT_X_AXIS_UNIT_REGEX, true);
243 243 }
244 244
245 245 void ScalarParserHelper::readResultLine(const QString &line)
246 246 {
247 247 tryReadResult(m_XAxisData, m_ValuesData, line, valuesIndexes());
248 248 }
249 249
250 250 std::vector<int> ScalarParserHelper::valuesIndexes() const
251 251 {
252 252 // Only one value to read
253 253 static auto result = std::vector<int>{0};
254 254 return result;
255 255 }
256 256
257 257 // /////////////////////// //
258 258 // SpectrogramParserHelper //
259 259 // /////////////////////// //
260 260
261 261 bool SpectrogramParserHelper::checkProperties()
262 262 {
263 263 // Generates y-axis data from bands extracted (take the middle of the intervals)
264 264 auto minBands = m_Properties.value(MIN_BANDS_PROPERTY).value<std::vector<double> >();
265 265 auto maxBands = m_Properties.value(MAX_BANDS_PROPERTY).value<std::vector<double> >();
266 266
267 267 if (minBands.size() != maxBands.size()) {
268 268 qCWarning(LOG_AmdaResultParserHelper()) << QObject::tr(
269 269 "Can't generate y-axis data from bands extracted: bands intervals are invalid");
270 270 return false;
271 271 }
272 272
273 273 std::transform(
274 274 minBands.begin(), minBands.end(), maxBands.begin(), std::back_inserter(m_YAxisData),
275 275 [](const auto &minValue, const auto &maxValue) { return (minValue + maxValue) / 2.; });
276 276
277 277 // Generates values indexes, i.e. the order in which each value will be retrieved (in ascending
278 278 // order of the associated bands)
279 279 m_ValuesIndexes = SortUtils::sortPermutation(m_YAxisData, std::less<double>());
280 280
281 281 // Sorts y-axis data accoding to the ascending order
282 282 m_YAxisData = SortUtils::sort(m_YAxisData, 1, m_ValuesIndexes);
283 283
284 284 // Sets fill value
285 285 m_FillValue = m_Properties.value(FILL_VALUE_PROPERTY).value<double>();
286 286
287 287 return true;
288 288 }
289 289
290 290 std::shared_ptr<IDataSeries> SpectrogramParserHelper::createSeries()
291 291 {
292 292 // Before creating the series, we handle its data holes
293 293 handleDataHoles();
294 294
295 295 return std::make_shared<SpectrogramSeries>(
296 296 std::move(m_XAxisData), std::move(m_YAxisData), std::move(m_ValuesData),
297 297 Unit{"t", true}, // x-axis unit is always a time unit
298 298 m_Properties.value(Y_AXIS_UNIT_PROPERTY).value<Unit>(),
299 299 m_Properties.value(VALUES_UNIT_PROPERTY).value<Unit>());
300 300 }
301 301
302 302 void SpectrogramParserHelper::readPropertyLine(const QString &line)
303 303 {
304 304 // Set of functions to test on the line to generate a property. If a function is valid (i.e. a
305 305 // property has been generated for the line), the line is treated as processed and the other
306 306 // functions are not called
307 307 std::vector<std::function<bool()> > functions{
308 308 // values unit
309 309 [&] {
310 310 return tryReadUnit(m_Properties, VALUES_UNIT_PROPERTY, line,
311 311 SPECTROGRAM_VALUES_UNIT_REGEX);
312 312 },
313 313 // y-axis unit
314 314 [&] {
315 315 return tryReadUnit(m_Properties, Y_AXIS_UNIT_PROPERTY, line,
316 316 SPECTROGRAM_Y_AXIS_UNIT_REGEX);
317 317 },
318 318 // min sampling
319 319 [&] {
320 320 return tryReadDouble(m_Properties, MIN_SAMPLING_PROPERTY, line,
321 321 SPECTROGRAM_MIN_SAMPLING_REGEX);
322 322 },
323 323 // max sampling
324 324 [&] {
325 325 return tryReadDouble(m_Properties, MAX_SAMPLING_PROPERTY, line,
326 326 SPECTROGRAM_MAX_SAMPLING_REGEX);
327 327 },
328 328 // fill value
329 329 [&] {
330 330 return tryReadDouble(m_Properties, FILL_VALUE_PROPERTY, line,
331 331 SPECTROGRAM_FILL_VALUE_REGEX);
332 332 },
333 333 // min bounds of each band
334 334 [&] {
335 335 return tryReadDoubles(m_Properties, MIN_BANDS_PROPERTY, line,
336 336 SPECTROGRAM_MIN_BANDS_REGEX);
337 337 },
338 338 // max bounds of each band
339 339 [&] {
340 340 return tryReadDoubles(m_Properties, MAX_BANDS_PROPERTY, line,
341 341 SPECTROGRAM_MAX_BANDS_REGEX);
342 342 }};
343 343
344 344 for (auto function : functions) {
345 345 // Stops at the first function that is valid
346 346 if (function()) {
347 347 return;
348 348 }
349 349 }
350 350 }
351 351
352 352 void SpectrogramParserHelper::readResultLine(const QString &line)
353 353 {
354 354 tryReadResult(m_XAxisData, m_ValuesData, line, m_ValuesIndexes, m_FillValue);
355 355 }
356 356
357 357 void SpectrogramParserHelper::handleDataHoles()
358 358 {
359 359 // Fills data holes according to the max resolution found in the AMDA file
360 360 auto resolution = m_Properties.value(MAX_SAMPLING_PROPERTY).value<double>();
361 361 auto fillValue = m_Properties.value(FILL_VALUE_PROPERTY).value<double>();
362 auto minBound = m_Properties.value(START_TIME_PROPERTY).value<double>();
363 auto maxBound = m_Properties.value(END_TIME_PROPERTY).value<double>();
362 364
363 DataSeriesUtils::fillDataHoles(m_XAxisData, m_ValuesData, resolution, fillValue);
365 DataSeriesUtils::fillDataHoles(m_XAxisData, m_ValuesData, resolution, fillValue, minBound,
366 maxBound);
364 367 }
365 368
366 369 // ////////////////// //
367 370 // VectorParserHelper //
368 371 // ////////////////// //
369 372
370 373 bool VectorParserHelper::checkProperties()
371 374 {
372 375 return checkUnit(m_Properties, X_AXIS_UNIT_PROPERTY,
373 376 QObject::tr("The x-axis unit could not be found in the file"));
374 377 }
375 378
376 379 std::shared_ptr<IDataSeries> VectorParserHelper::createSeries()
377 380 {
378 381 return std::make_shared<VectorSeries>(std::move(m_XAxisData), std::move(m_ValuesData),
379 382 m_Properties.value(X_AXIS_UNIT_PROPERTY).value<Unit>(),
380 383 m_Properties.value(VALUES_UNIT_PROPERTY).value<Unit>());
381 384 }
382 385
383 386 void VectorParserHelper::readPropertyLine(const QString &line)
384 387 {
385 388 tryReadUnit(m_Properties, X_AXIS_UNIT_PROPERTY, line, DEFAULT_X_AXIS_UNIT_REGEX, true);
386 389 }
387 390
388 391 void VectorParserHelper::readResultLine(const QString &line)
389 392 {
390 393 tryReadResult(m_XAxisData, m_ValuesData, line, valuesIndexes());
391 394 }
392 395
393 396 std::vector<int> VectorParserHelper::valuesIndexes() const
394 397 {
395 398 // 3 values to read, in order in the file (x, y, z)
396 399 static auto result = std::vector<int>{0, 1, 2};
397 400 return result;
398 401 }
General Comments 0
You need to be logged in to leave comments. Login now