##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Handles data holes in AMDA parser
Alexandre Leroux -
r980:c90f619c0c69
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,105 +1,107
1 1 #ifndef SCIQLOP_AMDARESULTPARSERHELPER_H
2 2 #define SCIQLOP_AMDARESULTPARSERHELPER_H
3 3
4 4 #include "AmdaResultParserDefs.h"
5 5
6 6 #include <QtCore/QLoggingCategory>
7 7 #include <QtCore/QString>
8 8
9 9 #include <memory>
10 10
11 11 class IDataSeries;
12 12
13 13 Q_DECLARE_LOGGING_CATEGORY(LOG_AmdaResultParserHelper)
14 14
15 15 /**
16 16 * Helper used to interpret the data of an AMDA result file and generate the corresponding data
17 17 * series.
18 18 *
19 19 * It proposes methods allowing to read line by line an AMDA file and to extract the properties
20 20 * (from the header) and the values corresponding to the data series
21 21 *
22 22 * @sa DataSeries
23 23 */
24 24 struct IAmdaResultParserHelper {
25 25 virtual ~IAmdaResultParserHelper() noexcept = default;
26 26
27 27 /// Verifies that the extracted properties are well formed and possibly applies other treatments
28 28 /// on them
29 29 /// @return true if the properties are well formed, false otherwise
30 30 virtual bool checkProperties() = 0;
31 31
32 32 /// Creates the data series from the properties and values extracted from the AMDA file.
33 33 /// @warning as the data are moved in the data series, the helper shouldn't be used after
34 34 /// calling this method
35 35 /// @return the data series created
36 36 virtual std::shared_ptr<IDataSeries> createSeries() = 0;
37 37
38 38 /// Reads a line from the AMDA file to extract a property that will be used to generate the data
39 39 /// series
40 40 /// @param line tahe line to interpret
41 41 virtual void readPropertyLine(const QString &line) = 0;
42 42
43 43 /// Reads a line from the AMDA file to extract a value that will be set in the data series
44 44 /// @param line the line to interpret
45 45 virtual void readResultLine(const QString &line) = 0;
46 46 };
47 47
48 48 /**
49 49 * Implementation of @sa IAmdaResultParserHelper for scalars
50 50 */
51 51 class ScalarParserHelper : public IAmdaResultParserHelper {
52 52 public:
53 53 bool checkProperties() override;
54 54 std::shared_ptr<IDataSeries> createSeries() override;
55 55 void readPropertyLine(const QString &line) override;
56 56 void readResultLine(const QString &line) override;
57 57
58 58 private:
59 59 /// @return the reading order of the "value" columns for a result line of the AMDA file
60 60 std::vector<int> valuesIndexes() const;
61 61
62 62 Properties m_Properties{};
63 63 std::vector<double> m_XAxisData{};
64 64 std::vector<double> m_ValuesData{};
65 65 };
66 66
67 67 /**
68 68 * Implementation of @sa IAmdaResultParserHelper for spectrograms
69 69 */
70 70 class SpectrogramParserHelper : public IAmdaResultParserHelper {
71 71 public:
72 72 bool checkProperties() override;
73 73 std::shared_ptr<IDataSeries> createSeries() override;
74 74 void readPropertyLine(const QString &line) override;
75 75 void readResultLine(const QString &line) override;
76 76
77 77 private:
78 void handleDataHoles();
79
78 80 Properties m_Properties{};
79 81 std::vector<double> m_XAxisData{};
80 82 std::vector<double> m_YAxisData{};
81 83 std::vector<double> m_ValuesData{};
82 84 std::vector<int> m_ValuesIndexes{};
83 85 double m_FillValue{std::numeric_limits<double>::quiet_NaN()};
84 86 };
85 87
86 88 /**
87 89 * Implementation of @sa IAmdaResultParserHelper for vectors
88 90 */
89 91 class VectorParserHelper : public IAmdaResultParserHelper {
90 92 public:
91 93 bool checkProperties() override;
92 94 std::shared_ptr<IDataSeries> createSeries() override;
93 95 void readPropertyLine(const QString &line) override;
94 96 void readResultLine(const QString &line) override;
95 97
96 98 private:
97 99 /// @return the reading order of the "value" columns for a result line of the AMDA file
98 100 std::vector<int> valuesIndexes() const;
99 101
100 102 Properties m_Properties{};
101 103 std::vector<double> m_XAxisData{};
102 104 std::vector<double> m_ValuesData{};
103 105 };
104 106
105 107 #endif // SCIQLOP_AMDARESULTPARSERHELPER_H
Show file before | Show file after | Hide comments
@@ -1,387 +1,398
1 1 #include "AmdaResultParserHelper.h"
2 2
3 3 #include <Common/DateUtils.h>
4 4 #include <Common/SortUtils.h>
5 5
6 #include <Data/DataSeriesUtils.h>
6 7 #include <Data/ScalarSeries.h>
7 8 #include <Data/SpectrogramSeries.h>
8 9 #include <Data/Unit.h>
9 10 #include <Data/VectorSeries.h>
10 11
11 12 #include <QtCore/QDateTime>
12 13 #include <QtCore/QRegularExpression>
13 14
14 15 #include <functional>
15 16
16 17 Q_LOGGING_CATEGORY(LOG_AmdaResultParserHelper, "AmdaResultParserHelper")
17 18
18 19 namespace {
19 20
20 21 // ///////// //
21 22 // Constants //
22 23 // ///////// //
23 24
24 25 /// Separator between values in a result line
25 26 const auto RESULT_LINE_SEPARATOR = QRegularExpression{QStringLiteral("\\s+")};
26 27
27 28 /// Format for dates in result files
28 29 const auto DATE_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss.zzz");
29 30
30 31 // /////// //
31 32 // Methods //
32 33 // /////// //
33 34
34 35 /**
35 36 * Checks that the properties contain a specific unit and that this unit is valid
36 37 * @param properties the properties map in which to search unit
37 38 * @param key the key to search for the unit in the properties
38 39 * @param errorMessage the error message to log in case the unit is invalid
39 40 * @return true if the unit is valid, false it it's invalid or was not found in the properties
40 41 */
41 42 bool checkUnit(const Properties &properties, const QString &key, const QString &errorMessage)
42 43 {
43 44 auto unit = properties.value(key).value<Unit>();
44 45 if (unit.m_Name.isEmpty()) {
45 46 qCWarning(LOG_AmdaResultParserHelper()) << errorMessage;
46 47 return false;
47 48 }
48 49
49 50 return true;
50 51 }
51 52
52 53 QDateTime dateTimeFromString(const QString &stringDate) noexcept
53 54 {
54 55 #if QT_VERSION >= QT_VERSION_CHECK(5, 8, 0)
55 56 return QDateTime::fromString(stringDate, Qt::ISODateWithMs);
56 57 #else
57 58 return QDateTime::fromString(stringDate, DATE_FORMAT);
58 59 #endif
59 60 }
60 61
61 62 /// Converts a string date to a double date
62 63 /// @return a double that represents the date in seconds, NaN if the string date can't be converted
63 64 double doubleDate(const QString &stringDate) noexcept
64 65 {
65 66 // Format: yyyy-MM-ddThh:mm:ss.zzz
66 67 auto dateTime = dateTimeFromString(stringDate);
67 68 dateTime.setTimeSpec(Qt::UTC);
68 69 return dateTime.isValid() ? DateUtils::secondsSinceEpoch(dateTime)
69 70 : std::numeric_limits<double>::quiet_NaN();
70 71 }
71 72
72 73 /**
73 74 * Reads a line from the AMDA file and tries to extract a x-axis data and value data from it
74 75 * @param xAxisData the vector in which to store the x-axis data extracted
75 76 * @param valuesData the vector in which to store the value extracted
76 77 * @param line the line to read to extract the property
77 78 * @param valuesIndexes indexes of insertion of read values. For example, if the line contains three
78 79 * columns of values, and valuesIndexes are {2, 0, 1}, the value of the third column will be read
79 80 * and inserted first, then the value of the first column, and finally the value of the second
80 81 * column.
81 82 * @param fillValue value that tags an invalid data. For example, if fillValue is -1 and a read
82 83 * value is -1, then this value is considered as invalid and converted to NaN
83 84 */
84 85 void tryReadResult(std::vector<double> &xAxisData, std::vector<double> &valuesData,
85 86 const QString &line, const std::vector<int> &valuesIndexes,
86 87 double fillValue = std::numeric_limits<double>::quiet_NaN())
87 88 {
88 89 auto lineData = line.split(RESULT_LINE_SEPARATOR, QString::SkipEmptyParts);
89 90
90 91 // Checks that the line contains expected number of values + x-axis value
91 92 if (static_cast<size_t>(lineData.size()) == valuesIndexes.size() + 1) {
92 93 // X : the data is converted from date to double (in secs)
93 94 auto x = doubleDate(lineData.at(0));
94 95
95 96 // Adds result only if x is valid. Then, if value is invalid, it is set to NaN
96 97 if (!std::isnan(x)) {
97 98 xAxisData.push_back(x);
98 99
99 100 // Values
100 101 for (auto valueIndex : valuesIndexes) {
101 102 bool valueOk;
102 103 // we use valueIndex + 1 to skip column 0 (x-axis value)
103 104 auto value = lineData.at(valueIndex + 1).toDouble(&valueOk);
104 105
105 106 if (!valueOk) {
106 107 qCWarning(LOG_AmdaResultParserHelper())
107 108 << QObject::tr(
108 109 "Value from (line %1, column %2) is invalid and will be "
109 110 "converted to NaN")
110 111 .arg(line, valueIndex);
111 112 value = std::numeric_limits<double>::quiet_NaN();
112 113 }
113 114
114 115 // Handles fill value
115 116 if (!std::isnan(fillValue) && !std::isnan(value) && fillValue == value) {
116 117 value = std::numeric_limits<double>::quiet_NaN();
117 118 }
118 119
119 120 valuesData.push_back(value);
120 121 }
121 122 }
122 123 else {
123 124 qCWarning(LOG_AmdaResultParserHelper())
124 125 << QObject::tr("Can't retrieve results from line %1: x is invalid").arg(line);
125 126 }
126 127 }
127 128 else {
128 129 qCWarning(LOG_AmdaResultParserHelper())
129 130 << QObject::tr("Can't retrieve results from line %1: invalid line").arg(line);
130 131 }
131 132 }
132 133
133 134 /**
134 135 * Reads a line from the AMDA file and tries to extract a property from it
135 136 * @param properties the properties map in which to put the property extracted from the line
136 137 * @param key the key to which the property is added in the properties map
137 138 * @param line the line to read to extract the property
138 139 * @param regex the expected regex to extract the property. If the line matches this regex, the
139 140 * property is generated
140 141 * @param fun the function used to generate the property
141 142 * @return true if the property could be generated, false if the line does not match the regex, or
142 143 * if a property has already been generated for the key
143 144 */
144 145 template <typename GeneratePropertyFun>
145 146 bool tryReadProperty(Properties &properties, const QString &key, const QString &line,
146 147 const QRegularExpression &regex, GeneratePropertyFun fun)
147 148 {
148 149 if (properties.contains(key)) {
149 150 return false;
150 151 }
151 152
152 153 auto match = regex.match(line);
153 154 if (match.hasMatch()) {
154 155 properties.insert(key, fun(match));
155 156 }
156 157
157 158 return match.hasMatch();
158 159 }
159 160
160 161 /**
161 162 * Reads a line from the AMDA file and tries to extract a double from it
162 163 * @sa tryReadProperty()
163 164 */
164 165 bool tryReadDouble(Properties &properties, const QString &key, const QString &line,
165 166 const QRegularExpression &regex)
166 167 {
167 168 return tryReadProperty(properties, key, line, regex, [](const auto &match) {
168 169 bool ok;
169 170
170 171 // If the value can't be converted to double, it is set to NaN
171 172 auto doubleValue = match.captured(1).toDouble(&ok);
172 173 if (!ok) {
173 174 doubleValue = std::numeric_limits<double>::quiet_NaN();
174 175 }
175 176
176 177 return QVariant::fromValue(doubleValue);
177 178 });
178 179 }
179 180
180 181 /**
181 182 * Reads a line from the AMDA file and tries to extract a vector of doubles from it
182 183 * @param sep the separator of double values in the line
183 184 * @sa tryReadProperty()
184 185 */
185 186 bool tryReadDoubles(Properties &properties, const QString &key, const QString &line,
186 187 const QRegularExpression &regex, const QString &sep = QStringLiteral(","))
187 188 {
188 189 return tryReadProperty(properties, key, line, regex, [sep](const auto &match) {
189 190 std::vector<double> doubleValues{};
190 191
191 192 // If the value can't be converted to double, it is set to NaN
192 193 auto values = match.captured(1).split(sep);
193 194 for (auto value : values) {
194 195 bool ok;
195 196
196 197 auto doubleValue = value.toDouble(&ok);
197 198 if (!ok) {
198 199 doubleValue = std::numeric_limits<double>::quiet_NaN();
199 200 }
200 201
201 202 doubleValues.push_back(doubleValue);
202 203 }
203 204
204 205 return QVariant::fromValue(doubleValues);
205 206 });
206 207 }
207 208
208 209 /**
209 210 * Reads a line from the AMDA file and tries to extract a unit from it
210 211 * @sa tryReadProperty()
211 212 */
212 213 bool tryReadUnit(Properties &properties, const QString &key, const QString &line,
213 214 const QRegularExpression &regex, bool timeUnit = false)
214 215 {
215 216 return tryReadProperty(properties, key, line, regex, [timeUnit](const auto &match) {
216 217 return QVariant::fromValue(Unit{match.captured(1), timeUnit});
217 218 });
218 219 }
219 220
220 221 } // namespace
221 222
222 223 // ////////////////// //
223 224 // ScalarParserHelper //
224 225 // ////////////////// //
225 226
226 227 bool ScalarParserHelper::checkProperties()
227 228 {
228 229 return checkUnit(m_Properties, X_AXIS_UNIT_PROPERTY,
229 230 QObject::tr("The x-axis unit could not be found in the file"));
230 231 }
231 232
232 233 std::shared_ptr<IDataSeries> ScalarParserHelper::createSeries()
233 234 {
234 235 return std::make_shared<ScalarSeries>(std::move(m_XAxisData), std::move(m_ValuesData),
235 236 m_Properties.value(X_AXIS_UNIT_PROPERTY).value<Unit>(),
236 237 m_Properties.value(VALUES_UNIT_PROPERTY).value<Unit>());
237 238 }
238 239
239 240 void ScalarParserHelper::readPropertyLine(const QString &line)
240 241 {
241 242 tryReadUnit(m_Properties, X_AXIS_UNIT_PROPERTY, line, DEFAULT_X_AXIS_UNIT_REGEX, true);
242 243 }
243 244
244 245 void ScalarParserHelper::readResultLine(const QString &line)
245 246 {
246 247 tryReadResult(m_XAxisData, m_ValuesData, line, valuesIndexes());
247 248 }
248 249
249 250 std::vector<int> ScalarParserHelper::valuesIndexes() const
250 251 {
251 252 // Only one value to read
252 253 static auto result = std::vector<int>{0};
253 254 return result;
254 255 }
255 256
256 257 // /////////////////////// //
257 258 // SpectrogramParserHelper //
258 259 // /////////////////////// //
259 260
260 261 bool SpectrogramParserHelper::checkProperties()
261 262 {
262 263 // Generates y-axis data from bands extracted (take the middle of the intervals)
263 264 auto minBands = m_Properties.value(MIN_BANDS_PROPERTY).value<std::vector<double> >();
264 265 auto maxBands = m_Properties.value(MAX_BANDS_PROPERTY).value<std::vector<double> >();
265 266
266 267 if (minBands.size() != maxBands.size()) {
267 268 qCWarning(LOG_AmdaResultParserHelper()) << QObject::tr(
268 269 "Can't generate y-axis data from bands extracted: bands intervals are invalid");
269 270 return false;
270 271 }
271 272
272 273 std::transform(
273 274 minBands.begin(), minBands.end(), maxBands.begin(), std::back_inserter(m_YAxisData),
274 275 [](const auto &minValue, const auto &maxValue) { return (minValue + maxValue) / 2.; });
275 276
276 277 // Generates values indexes, i.e. the order in which each value will be retrieved (in ascending
277 278 // order of the associated bands)
278 279 m_ValuesIndexes = SortUtils::sortPermutation(m_YAxisData, std::less<double>());
279 280
280 281 // Sorts y-axis data accoding to the ascending order
281 282 m_YAxisData = SortUtils::sort(m_YAxisData, 1, m_ValuesIndexes);
282 283
283 284 // Sets fill value
284 285 m_FillValue = m_Properties.value(FILL_VALUE_PROPERTY).value<double>();
285 286
286 /// @todo: handle min/max samplings?
287
288 287 return true;
289 288 }
290 289
291 290 std::shared_ptr<IDataSeries> SpectrogramParserHelper::createSeries()
292 291 {
292 // Before creating the series, we handle its data holes
293 handleDataHoles();
294
293 295 return std::make_shared<SpectrogramSeries>(
294 296 std::move(m_XAxisData), std::move(m_YAxisData), std::move(m_ValuesData),
295 297 Unit{"t", true}, // x-axis unit is always a time unit
296 298 m_Properties.value(Y_AXIS_UNIT_PROPERTY).value<Unit>(),
297 299 m_Properties.value(VALUES_UNIT_PROPERTY).value<Unit>());
298 300 }
299 301
300 302 void SpectrogramParserHelper::readPropertyLine(const QString &line)
301 303 {
302 304 // Set of functions to test on the line to generate a property. If a function is valid (i.e. a
303 305 // property has been generated for the line), the line is treated as processed and the other
304 306 // functions are not called
305 307 std::vector<std::function<bool()> > functions{
306 308 // values unit
307 309 [&] {
308 310 return tryReadUnit(m_Properties, VALUES_UNIT_PROPERTY, line,
309 311 SPECTROGRAM_VALUES_UNIT_REGEX);
310 312 },
311 313 // y-axis unit
312 314 [&] {
313 315 return tryReadUnit(m_Properties, Y_AXIS_UNIT_PROPERTY, line,
314 316 SPECTROGRAM_Y_AXIS_UNIT_REGEX);
315 317 },
316 318 // min sampling
317 319 [&] {
318 320 return tryReadDouble(m_Properties, MIN_SAMPLING_PROPERTY, line,
319 321 SPECTROGRAM_MIN_SAMPLING_REGEX);
320 322 },
321 323 // max sampling
322 324 [&] {
323 325 return tryReadDouble(m_Properties, MAX_SAMPLING_PROPERTY, line,
324 326 SPECTROGRAM_MAX_SAMPLING_REGEX);
325 327 },
326 328 // fill value
327 329 [&] {
328 330 return tryReadDouble(m_Properties, FILL_VALUE_PROPERTY, line,
329 331 SPECTROGRAM_FILL_VALUE_REGEX);
330 332 },
331 333 // min bounds of each band
332 334 [&] {
333 335 return tryReadDoubles(m_Properties, MIN_BANDS_PROPERTY, line,
334 336 SPECTROGRAM_MIN_BANDS_REGEX);
335 337 },
336 338 // max bounds of each band
337 339 [&] {
338 340 return tryReadDoubles(m_Properties, MAX_BANDS_PROPERTY, line,
339 341 SPECTROGRAM_MAX_BANDS_REGEX);
340 342 }};
341 343
342 344 for (auto function : functions) {
343 345 // Stops at the first function that is valid
344 346 if (function()) {
345 347 return;
346 348 }
347 349 }
348 350 }
349 351
350 352 void SpectrogramParserHelper::readResultLine(const QString &line)
351 353 {
352 354 tryReadResult(m_XAxisData, m_ValuesData, line, m_ValuesIndexes, m_FillValue);
353 355 }
354 356
357 void SpectrogramParserHelper::handleDataHoles()
358 {
359 // Fills data holes according to the max resolution found in the AMDA file
360 auto resolution = m_Properties.value(MAX_SAMPLING_PROPERTY).value<double>();
361 auto fillValue = m_Properties.value(FILL_VALUE_PROPERTY).value<double>();
362
363 DataSeriesUtils::fillDataHoles(m_XAxisData, m_ValuesData, resolution, fillValue);
364 }
365
355 366 // ////////////////// //
356 367 // VectorParserHelper //
357 368 // ////////////////// //
358 369
359 370 bool VectorParserHelper::checkProperties()
360 371 {
361 372 return checkUnit(m_Properties, X_AXIS_UNIT_PROPERTY,
362 373 QObject::tr("The x-axis unit could not be found in the file"));
363 374 }
364 375
365 376 std::shared_ptr<IDataSeries> VectorParserHelper::createSeries()
366 377 {
367 378 return std::make_shared<VectorSeries>(std::move(m_XAxisData), std::move(m_ValuesData),
368 379 m_Properties.value(X_AXIS_UNIT_PROPERTY).value<Unit>(),
369 380 m_Properties.value(VALUES_UNIT_PROPERTY).value<Unit>());
370 381 }
371 382
372 383 void VectorParserHelper::readPropertyLine(const QString &line)
373 384 {
374 385 tryReadUnit(m_Properties, X_AXIS_UNIT_PROPERTY, line, DEFAULT_X_AXIS_UNIT_REGEX, true);
375 386 }
376 387
377 388 void VectorParserHelper::readResultLine(const QString &line)
378 389 {
379 390 tryReadResult(m_XAxisData, m_ValuesData, line, valuesIndexes());
380 391 }
381 392
382 393 std::vector<int> VectorParserHelper::valuesIndexes() const
383 394 {
384 395 // 3 values to read, in order in the file (x, y, z)
385 396 static auto result = std::vector<int>{0, 1, 2};
386 397 return result;
387 398 }
General Comments 0
You need to be logged in to leave comments. Login now