##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Merge branch 'feature/HandleNaNAmda' into develop
Alexandre Leroux -
r498:3b5523f881eb merge
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,6
1 #Sampling Time : 60
2 #Time Format : YYYY-MM-DDThh:mm:ss.mls
3 #imf(0) - Type : Local Parameter @ CDPP/AMDA - Name : bx_gse - Units : nT - Size : 1 - Frame : GSE - Mission : ACE - Instrument : MFI - Dataset : mfi_final-prelim
4 NaN -3.01425
5 2013-09-23T09:01:30.000 -2.71850
6 2013-09-23T09:02:30.000 -2.52150 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,152 +1,160
1 1 #include "AmdaResultParser.h"
2 2
3 3 #include <Common/DateUtils.h>
4 4 #include <Data/ScalarSeries.h>
5 5
6 6 #include <QDateTime>
7 7 #include <QFile>
8 8 #include <QRegularExpression>
9 9
10 10 #include <cmath>
11 11
12 12 Q_LOGGING_CATEGORY(LOG_AmdaResultParser, "AmdaResultParser")
13 13
14 14 namespace {
15 15
16 16 /// Message in result file when the file was not found on server
17 17 const auto FILE_NOT_FOUND_MESSAGE = QStringLiteral("Not Found");
18 18
19 19 /// Format for dates in result files
20 20 const auto DATE_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss.zzz");
21 21
22 22 /// Separator between values in a result line
23 23 const auto RESULT_LINE_SEPARATOR = QRegularExpression{QStringLiteral("\\s+")};
24 24
25 25 /// Regex to find unit in a line. Examples of valid lines:
26 26 /// ... - Units : nT - ...
27 27 /// ... -Units:nT- ...
28 28 /// ... -Units: mΒ²- ...
29 29 /// ... - Units : m/s - ...
30 30 const auto UNIT_REGEX = QRegularExpression{QStringLiteral("-\\s*Units\\s*:\\s*(.+?)\\s*-")};
31 31
32 32 /// Converts a string date to a double date
33 33 /// @return a double that represents the date in seconds, NaN if the string date can't be converted
34 34 double doubleDate(const QString &stringDate) noexcept
35 35 {
36 36 auto dateTime = QDateTime::fromString(stringDate, DATE_FORMAT);
37 37 dateTime.setTimeSpec(Qt::UTC);
38 38 return dateTime.isValid() ? DateUtils::secondsSinceEpoch(dateTime)
39 39 : std::numeric_limits<double>::quiet_NaN();
40 40 }
41 41
42 42 /// Checks if a line is a comment line
43 43 bool isCommentLine(const QString &line)
44 44 {
45 45 return line.startsWith("#");
46 46 }
47 47
48 48 /**
49 49 * Reads stream to retrieve x-axis unit
50 50 * @param stream the stream to read
51 51 * @return the unit that has been read in the stream, a default unit (time unit with no label) if an
52 52 * error occured during reading
53 53 */
54 54 Unit readXAxisUnit(QTextStream &stream)
55 55 {
56 56 QString line{};
57 57
58 58 // Searches unit in the comment lines
59 59 while (stream.readLineInto(&line) && isCommentLine(line)) {
60 60 auto match = UNIT_REGEX.match(line);
61 61 if (match.hasMatch()) {
62 62 return Unit{match.captured(1), true};
63 63 }
64 64 }
65 65
66 66 qCWarning(LOG_AmdaResultParser()) << QObject::tr("The unit could not be found in the file");
67 67
68 68 // Error cases
69 69 return Unit{{}, true};
70 70 }
71 71
72 72 /**
73 73 * Reads stream to retrieve results
74 74 * @param stream the stream to read
75 75 * @return the pair of vectors x-axis data/values data that has been read in the stream
76 76 */
77 77 QPair<QVector<double>, QVector<double> > readResults(QTextStream &stream)
78 78 {
79 79 auto xData = QVector<double>{};
80 80 auto valuesData = QVector<double>{};
81 81
82 82 QString line{};
83 83
84 84 while (stream.readLineInto(&line)) {
85 85 // Ignore comment lines
86 86 if (!isCommentLine(line)) {
87 87 auto lineData = line.split(RESULT_LINE_SEPARATOR, QString::SkipEmptyParts);
88 88 if (lineData.size() == 2) {
89 89 // X : the data is converted from date to double (in secs)
90 90 auto x = doubleDate(lineData.at(0));
91 91
92 92 // Value
93 93 bool valueOk;
94 94 auto value = lineData.at(1).toDouble(&valueOk);
95 95
96 // Adds result only if x and value are valid
97 if (!std::isnan(x) && !std::isnan(value) && valueOk) {
96 // Adds result only if x is valid. Then, if value is invalid, it is set to NaN
97 if (!std::isnan(x)) {
98 98 xData.push_back(x);
99
100 if (!valueOk) {
101 qCWarning(LOG_AmdaResultParser())
102 << QObject::tr(
103 "Value from line %1 is invalid and will be converted to NaN")
104 .arg(line);
105 value = std::numeric_limits<double>::quiet_NaN();
106 }
107
99 108 valuesData.push_back(value);
100 109 }
101 110 else {
102 111 qCWarning(LOG_AmdaResultParser())
103 << QObject::tr(
104 "Can't retrieve results from line %1: x and/or value are invalid")
112 << QObject::tr("Can't retrieve results from line %1: x is invalid")
105 113 .arg(line);
106 114 }
107 115 }
108 116 else {
109 117 qCWarning(LOG_AmdaResultParser())
110 118 << QObject::tr("Can't retrieve results from line %1: invalid line").arg(line);
111 119 }
112 120 }
113 121 }
114 122
115 123 return qMakePair(std::move(xData), std::move(valuesData));
116 124 }
117 125
118 126 } // namespace
119 127
120 128 std::shared_ptr<IDataSeries> AmdaResultParser::readTxt(const QString &filePath) noexcept
121 129 {
122 130 QFile file{filePath};
123 131
124 132 if (!file.open(QFile::ReadOnly | QIODevice::Text)) {
125 133 qCCritical(LOG_AmdaResultParser())
126 134 << QObject::tr("Can't retrieve AMDA data from file %1: %2")
127 135 .arg(filePath, file.errorString());
128 136 return nullptr;
129 137 }
130 138
131 139 QTextStream stream{&file};
132 140
133 141 // Checks if the file was found on the server
134 142 auto firstLine = stream.readLine();
135 143 if (firstLine.compare(FILE_NOT_FOUND_MESSAGE) == 0) {
136 144 qCCritical(LOG_AmdaResultParser())
137 145 << QObject::tr("Can't retrieve AMDA data from file %1: file was not found on server")
138 146 .arg(filePath);
139 147 return nullptr;
140 148 }
141 149
142 150 // Reads x-axis unit
143 151 stream.seek(0); // returns to the beginning of the file
144 152 auto xAxisUnit = readXAxisUnit(stream);
145 153
146 154 // Reads results
147 155 stream.seek(0); // returns to the beginning of the file
148 156 auto results = readResults(stream);
149 157
150 158 return std::make_shared<ScalarSeries>(std::move(results.first), std::move(results.second),
151 159 xAxisUnit, Unit{});
152 160 }
Show file before | Show file after | Hide comments
@@ -1,182 +1,212
1 1 #include "AmdaResultParser.h"
2 2
3 3 #include <Data/ScalarSeries.h>
4 4
5 5 #include <QObject>
6 6 #include <QtTest>
7 7
8 8 namespace {
9 9
10 10 /// Path for the tests
11 11 const auto TESTS_RESOURCES_PATH
12 12 = QFileInfo{QString{AMDA_TESTS_RESOURCES_DIR}, "TestAmdaResultParser"}.absoluteFilePath();
13 13
14 /// Compares two vectors that can potentially contain NaN values
15 bool compareVectors(const QVector<double> &v1, const QVector<double> &v2)
16 {
17 if (v1.size() != v2.size()) {
18 return false;
19 }
20
21 auto result = true;
22 auto v2It = v2.cbegin();
23 for (auto v1It = v1.cbegin(), v1End = v1.cend(); v1It != v1End && result; ++v1It, ++v2It) {
24 auto v1Value = *v1It;
25 auto v2Value = *v2It;
26
27 // If v1 is NaN, v2 has to be NaN too
28 result = std::isnan(v1Value) ? std::isnan(v2Value) : (v1Value == v2Value);
29 }
30
31 return result;
32 }
33
14 34 QString inputFilePath(const QString &inputFileName)
15 35 {
16 36 return QFileInfo{TESTS_RESOURCES_PATH, inputFileName}.absoluteFilePath();
17 37 }
18 38
19 39 struct ExpectedResults {
20 40 explicit ExpectedResults() = default;
21 41
22 42 /// Ctor with QVector<QDateTime> as x-axis data. Datetimes are converted to doubles
23 43 explicit ExpectedResults(Unit xAxisUnit, Unit valuesUnit, const QVector<QDateTime> &xAxisData,
24 44 QVector<double> valuesData)
25 45 : m_ParsingOK{true},
26 46 m_XAxisUnit{xAxisUnit},
27 47 m_ValuesUnit{valuesUnit},
28 48 m_XAxisData{},
29 49 m_ValuesData{std::move(valuesData)}
30 50 {
31 51 // Converts QVector<QDateTime> to QVector<double>
32 52 std::transform(xAxisData.cbegin(), xAxisData.cend(), std::back_inserter(m_XAxisData),
33 53 [](const auto &dateTime) { return dateTime.toMSecsSinceEpoch() / 1000.; });
34 54 }
35 55
36 56 /**
37 57 * Validates a DataSeries compared to the expected results
38 58 * @param results the DataSeries to validate
39 59 */
40 60 void validate(std::shared_ptr<IDataSeries> results)
41 61 {
42 62 if (m_ParsingOK) {
43 63 auto scalarSeries = dynamic_cast<ScalarSeries *>(results.get());
44 64 QVERIFY(scalarSeries != nullptr);
45 65
46 66 // Checks units
47 67 QVERIFY(scalarSeries->xAxisUnit() == m_XAxisUnit);
48 68 QVERIFY(scalarSeries->valuesUnit() == m_ValuesUnit);
49 69
50 // Checks values
51 QVERIFY(scalarSeries->xAxisData()->data() == m_XAxisData);
52 QVERIFY(scalarSeries->valuesData()->data() == m_ValuesData);
70 // Checks values : as the vectors can potentially contain NaN values, we must use a
71 // custom vector comparison method
72 QVERIFY(compareVectors(scalarSeries->xAxisData()->data(), m_XAxisData));
73 QVERIFY(compareVectors(scalarSeries->valuesData()->data(), m_ValuesData));
53 74 }
54 75 else {
55 76 QVERIFY(results == nullptr);
56 77 }
57 78 }
58 79
59 80 // Parsing was successfully completed
60 81 bool m_ParsingOK{false};
61 82 // Expected x-axis unit
62 83 Unit m_XAxisUnit{};
63 84 // Expected values unit
64 85 Unit m_ValuesUnit{};
65 86 // Expected x-axis data
66 87 QVector<double> m_XAxisData{};
67 88 // Expected values data
68 89 QVector<double> m_ValuesData{};
69 90 };
70 91
71 92 } // namespace
72 93
73 94 Q_DECLARE_METATYPE(ExpectedResults)
74 95
75 96 class TestAmdaResultParser : public QObject {
76 97 Q_OBJECT
77 98 private slots:
78 99 /// Input test data
79 100 /// @sa testTxtJson()
80 101 void testReadTxt_data();
81 102
82 103 /// Tests parsing of a TXT file
83 104 void testReadTxt();
84 105 };
85 106
86 107 void TestAmdaResultParser::testReadTxt_data()
87 108 {
88 109 // ////////////// //
89 110 // Test structure //
90 111 // ////////////// //
91 112
92 113 // Name of TXT file to read
93 114 QTest::addColumn<QString>("inputFileName");
94 115 // Expected results
95 116 QTest::addColumn<ExpectedResults>("expectedResults");
96 117
97 118 // ////////// //
98 119 // Test cases //
99 120 // ////////// //
100 121
101 122 auto dateTime = [](int year, int month, int day, int hours, int minutes, int seconds) {
102 123 return QDateTime{{year, month, day}, {hours, minutes, seconds}, Qt::UTC};
103 124 };
104 125
105 // Valid file
126 // Valid files
106 127 QTest::newRow("Valid file")
107 128 << QStringLiteral("ValidScalar1.txt")
108 129 << ExpectedResults{
109 130 Unit{QStringLiteral("nT"), true}, Unit{},
110 131 QVector<QDateTime>{dateTime(2013, 9, 23, 9, 0, 30), dateTime(2013, 9, 23, 9, 1, 30),
111 132 dateTime(2013, 9, 23, 9, 2, 30), dateTime(2013, 9, 23, 9, 3, 30),
112 133 dateTime(2013, 9, 23, 9, 4, 30), dateTime(2013, 9, 23, 9, 5, 30),
113 134 dateTime(2013, 9, 23, 9, 6, 30), dateTime(2013, 9, 23, 9, 7, 30),
114 135 dateTime(2013, 9, 23, 9, 8, 30), dateTime(2013, 9, 23, 9, 9, 30)},
115 136 QVector<double>{-2.83950, -2.71850, -2.52150, -2.57633, -2.58050, -2.48325, -2.63025,
116 137 -2.55800, -2.43250, -2.42200}};
117 138
139 QTest::newRow("Valid file (value of first line is invalid but it is converted to NaN")
140 << QStringLiteral("WrongValue.txt")
141 << ExpectedResults{
142 Unit{QStringLiteral("nT"), true}, Unit{},
143 QVector<QDateTime>{dateTime(2013, 9, 23, 9, 0, 30), dateTime(2013, 9, 23, 9, 1, 30),
144 dateTime(2013, 9, 23, 9, 2, 30)},
145 QVector<double>{std::numeric_limits<double>::quiet_NaN(), -2.71850, -2.52150}};
146
147 QTest::newRow("Valid file that contains NaN values")
148 << QStringLiteral("NaNValue.txt")
149 << ExpectedResults{
150 Unit{QStringLiteral("nT"), true}, Unit{},
151 QVector<QDateTime>{dateTime(2013, 9, 23, 9, 0, 30), dateTime(2013, 9, 23, 9, 1, 30),
152 dateTime(2013, 9, 23, 9, 2, 30)},
153 QVector<double>{std::numeric_limits<double>::quiet_NaN(), -2.71850, -2.52150}};
154
118 155 // Valid files but with some invalid lines (wrong unit, wrong values, etc.)
119 156 QTest::newRow("No unit file") << QStringLiteral("NoUnit.txt")
120 157 << ExpectedResults{Unit{QStringLiteral(""), true}, Unit{},
121 158 QVector<QDateTime>{}, QVector<double>{}};
122 159 QTest::newRow("Wrong unit file")
123 160 << QStringLiteral("WrongUnit.txt")
124 161 << ExpectedResults{Unit{QStringLiteral(""), true}, Unit{},
125 162 QVector<QDateTime>{dateTime(2013, 9, 23, 9, 0, 30),
126 163 dateTime(2013, 9, 23, 9, 1, 30),
127 164 dateTime(2013, 9, 23, 9, 2, 30)},
128 165 QVector<double>{-2.83950, -2.71850, -2.52150}};
129 166
130 167 QTest::newRow("Wrong results file (date of first line is invalid")
131 168 << QStringLiteral("WrongDate.txt")
132 169 << ExpectedResults{
133 170 Unit{QStringLiteral("nT"), true}, Unit{},
134 171 QVector<QDateTime>{dateTime(2013, 9, 23, 9, 1, 30), dateTime(2013, 9, 23, 9, 2, 30)},
135 172 QVector<double>{-2.71850, -2.52150}};
136 173
137 174 QTest::newRow("Wrong results file (too many values for first line")
138 175 << QStringLiteral("TooManyValues.txt")
139 176 << ExpectedResults{
140 177 Unit{QStringLiteral("nT"), true}, Unit{},
141 178 QVector<QDateTime>{dateTime(2013, 9, 23, 9, 1, 30), dateTime(2013, 9, 23, 9, 2, 30)},
142 179 QVector<double>{-2.71850, -2.52150}};
143 180
144 QTest::newRow("Wrong results file (value of first line is invalid")
145 << QStringLiteral("WrongValue.txt")
146 << ExpectedResults{
147 Unit{QStringLiteral("nT"), true}, Unit{},
148 QVector<QDateTime>{dateTime(2013, 9, 23, 9, 1, 30), dateTime(2013, 9, 23, 9, 2, 30)},
149 QVector<double>{-2.71850, -2.52150}};
150
151 QTest::newRow("Wrong results file (value of first line is NaN")
152 << QStringLiteral("NaNValue.txt")
181 QTest::newRow("Wrong results file (x of first line is NaN")
182 << QStringLiteral("NaNX.txt")
153 183 << ExpectedResults{
154 184 Unit{QStringLiteral("nT"), true}, Unit{},
155 185 QVector<QDateTime>{dateTime(2013, 9, 23, 9, 1, 30), dateTime(2013, 9, 23, 9, 2, 30)},
156 186 QVector<double>{-2.71850, -2.52150}};
157 187
158 188 // Invalid files
159 189 QTest::newRow("Invalid file (unexisting file)") << QStringLiteral("UnexistingFile.txt")
160 190 << ExpectedResults{};
161 191
162 192 QTest::newRow("Invalid file (file not found on server)") << QStringLiteral("FileNotFound.txt")
163 193 << ExpectedResults{};
164 194 }
165 195
166 196 void TestAmdaResultParser::testReadTxt()
167 197 {
168 198 QFETCH(QString, inputFileName);
169 199 QFETCH(ExpectedResults, expectedResults);
170 200
171 201 // Parses file
172 202 auto filePath = inputFilePath(inputFileName);
173 203 auto results = AmdaResultParser::readTxt(filePath);
174 204
175 205 // ///////////////// //
176 206 // Validates results //
177 207 // ///////////////// //
178 208 expectedResults.validate(results);
179 209 }
180 210
181 211 QTEST_MAIN(TestAmdaResultParser)
182 212 #include "TestAmdaResultParser.moc"
General Comments 0
You need to be logged in to leave comments. Login now