##// END OF EJS Templates
GIT_REPOSITORIES » LPP » Users » mperrinel » SCIQLOP-Initialisation
RSS

Fork of HG_REPOSITORIES/LPP/SciQLOP_Repos/SciQLop

Improves reading performances
Alexandre Leroux -
r727:4449c7d1b0d9
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,212 +1,213
1 #include "AmdaResultParser.h"
1 #include "AmdaResultParser.h"
2
2
3 #include <Common/DateUtils.h>
3 #include <Common/DateUtils.h>
4 #include <Data/ScalarSeries.h>
4 #include <Data/ScalarSeries.h>
5 #include <Data/VectorSeries.h>
5 #include <Data/VectorSeries.h>
6
6
7 #include <QDateTime>
7 #include <QDateTime>
8 #include <QFile>
8 #include <QFile>
9 #include <QRegularExpression>
9 #include <QRegularExpression>
10
10
11 #include <cmath>
11 #include <cmath>
12
12
13 Q_LOGGING_CATEGORY(LOG_AmdaResultParser, "AmdaResultParser")
13 Q_LOGGING_CATEGORY(LOG_AmdaResultParser, "AmdaResultParser")
14
14
15 namespace {
15 namespace {
16
16
17 /// Message in result file when the file was not found on server
17 /// Message in result file when the file was not found on server
18 const auto FILE_NOT_FOUND_MESSAGE = QStringLiteral("Not Found");
18 const auto FILE_NOT_FOUND_MESSAGE = QStringLiteral("Not Found");
19
19
20 /// Format for dates in result files
21 const auto DATE_FORMAT = QStringLiteral("yyyy-MM-ddThh:mm:ss.zzz");
22
23 /// Separator between values in a result line
20 /// Separator between values in a result line
24 const auto RESULT_LINE_SEPARATOR = QRegularExpression{QStringLiteral("\\s+")};
21 const auto RESULT_LINE_SEPARATOR = QRegularExpression{QStringLiteral("\\s+")};
25
22
26 /// Regex to find the header of the data in the file. This header indicates the end of comments in
23 /// Regex to find the header of the data in the file. This header indicates the end of comments in
27 /// the file
24 /// the file
28 const auto DATA_HEADER_REGEX = QRegularExpression{QStringLiteral("#\\s*DATA\\s*:")};
25 const auto DATA_HEADER_REGEX = QRegularExpression{QStringLiteral("#\\s*DATA\\s*:")};
29
26
30 /// Regex to find unit in a line. Examples of valid lines:
27 /// Regex to find unit in a line. Examples of valid lines:
31 /// ... PARAMETER_UNITS : nT ...
28 /// ... PARAMETER_UNITS : nT ...
32 /// ... PARAMETER_UNITS:nT ...
29 /// ... PARAMETER_UNITS:nT ...
33 /// ... PARAMETER_UNITS: mΒ² ...
30 /// ... PARAMETER_UNITS: mΒ² ...
34 /// ... PARAMETER_UNITS : m/s ...
31 /// ... PARAMETER_UNITS : m/s ...
35 const auto UNIT_REGEX = QRegularExpression{QStringLiteral("\\s*PARAMETER_UNITS\\s*:\\s*(.+)")};
32 const auto UNIT_REGEX = QRegularExpression{QStringLiteral("\\s*PARAMETER_UNITS\\s*:\\s*(.+)")};
36
33
37 /// Converts a string date to a double date
34 /// Converts a string date to a double date
38 /// @return a double that represents the date in seconds, NaN if the string date can't be converted
35 /// @return a double that represents the date in seconds, NaN if the string date can't be converted
39 double doubleDate(const QString &stringDate) noexcept
36 double doubleDate(const QString &stringDate) noexcept
40 {
37 {
41 auto dateTime = QDateTime::fromString(stringDate, DATE_FORMAT);
38 // Format: yyyy-MM-ddThh:mm:ss.zzz
39 auto dateTime = QDateTime::fromString(stringDate, Qt::ISODateWithMs);
42 dateTime.setTimeSpec(Qt::UTC);
40 dateTime.setTimeSpec(Qt::UTC);
43 return dateTime.isValid() ? DateUtils::secondsSinceEpoch(dateTime)
41 return dateTime.isValid() ? DateUtils::secondsSinceEpoch(dateTime)
44 : std::numeric_limits<double>::quiet_NaN();
42 : std::numeric_limits<double>::quiet_NaN();
45 }
43 }
46
44
47 /// Checks if a line is a comment line
45 /// Checks if a line is a comment line
48 bool isCommentLine(const QString &line)
46 bool isCommentLine(const QString &line)
49 {
47 {
50 return line.startsWith("#");
48 return line.startsWith("#");
51 }
49 }
52
50
53 /// @return the number of lines to be read depending on the type of value passed in parameter
51 /// @return the number of lines to be read depending on the type of value passed in parameter
54 int nbValues(AmdaResultParser::ValueType valueType) noexcept
52 int nbValues(AmdaResultParser::ValueType valueType) noexcept
55 {
53 {
56 switch (valueType) {
54 switch (valueType) {
57 case AmdaResultParser::ValueType::SCALAR:
55 case AmdaResultParser::ValueType::SCALAR:
58 return 1;
56 return 1;
59 case AmdaResultParser::ValueType::VECTOR:
57 case AmdaResultParser::ValueType::VECTOR:
60 return 3;
58 return 3;
61 case AmdaResultParser::ValueType::UNKNOWN:
59 case AmdaResultParser::ValueType::UNKNOWN:
62 // Invalid case
60 // Invalid case
63 break;
61 break;
64 }
62 }
65
63
66 // Invalid cases
64 // Invalid cases
67 qCCritical(LOG_AmdaResultParser())
65 qCCritical(LOG_AmdaResultParser())
68 << QObject::tr("Can't get the number of values to read: unsupported type");
66 << QObject::tr("Can't get the number of values to read: unsupported type");
69 return 0;
67 return 0;
70 }
68 }
71
69
72 /**
70 /**
73 * Reads stream to retrieve x-axis unit
71 * Reads stream to retrieve x-axis unit
74 * @param stream the stream to read
72 * @param stream the stream to read
75 * @return the unit that has been read in the stream, a default unit (time unit with no label) if an
73 * @return the unit that has been read in the stream, a default unit (time unit with no label) if an
76 * error occured during reading
74 * error occured during reading
77 */
75 */
78 Unit readXAxisUnit(QTextStream &stream)
76 Unit readXAxisUnit(QTextStream &stream)
79 {
77 {
80 QString line{};
78 QString line{};
81
79
82 // Searches unit in the comment lines (as long as the reading has not reached the data header)
80 // Searches unit in the comment lines (as long as the reading has not reached the data header)
83 while (stream.readLineInto(&line) && !line.contains(DATA_HEADER_REGEX)) {
81 while (stream.readLineInto(&line) && !line.contains(DATA_HEADER_REGEX)) {
84 auto match = UNIT_REGEX.match(line);
82 auto match = UNIT_REGEX.match(line);
85 if (match.hasMatch()) {
83 if (match.hasMatch()) {
86 return Unit{match.captured(1), true};
84 return Unit{match.captured(1), true};
87 }
85 }
88 }
86 }
89
87
90 qCWarning(LOG_AmdaResultParser()) << QObject::tr("The unit could not be found in the file");
88 qCWarning(LOG_AmdaResultParser()) << QObject::tr("The unit could not be found in the file");
91
89
92 // Error cases
90 // Error cases
93 return Unit{{}, true};
91 return Unit{{}, true};
94 }
92 }
95
93
96 /**
94 /**
97 * Reads stream to retrieve results
95 * Reads stream to retrieve results
98 * @param stream the stream to read
96 * @param stream the stream to read
99 * @return the pair of vectors x-axis data/values data that has been read in the stream
97 * @return the pair of vectors x-axis data/values data that has been read in the stream
100 */
98 */
101 std::pair<std::vector<double>, std::vector<double> >
99 std::pair<std::vector<double>, std::vector<double> >
102 readResults(QTextStream &stream, AmdaResultParser::ValueType valueType)
100 readResults(QTextStream &stream, AmdaResultParser::ValueType valueType)
103 {
101 {
104 auto expectedNbValues = nbValues(valueType);
102 auto expectedNbValues = nbValues(valueType) + 1;
105
103
106 auto xData = std::vector<double>{};
104 auto xData = std::vector<double>{};
107 auto valuesData = std::vector<double>{};
105 auto valuesData = std::vector<double>{};
108
106
109 QString line{};
107 QString line{};
110
108
111 while (stream.readLineInto(&line)) {
109 // Skip comment lines
112 // Ignore comment lines
110 while (stream.readLineInto(&line) && isCommentLine(line)) {
113 if (!isCommentLine(line)) {
111 }
112
113 if (!stream.atEnd()) {
114 do {
114 auto lineData = line.split(RESULT_LINE_SEPARATOR, QString::SkipEmptyParts);
115 auto lineData = line.split(RESULT_LINE_SEPARATOR, QString::SkipEmptyParts);
115 if (lineData.size() == expectedNbValues + 1) {
116 if (lineData.size() == expectedNbValues) {
116 // X : the data is converted from date to double (in secs)
117 // X : the data is converted from date to double (in secs)
117 auto x = doubleDate(lineData.at(0));
118 auto x = doubleDate(lineData.at(0));
118
119
119 // Adds result only if x is valid. Then, if value is invalid, it is set to NaN
120 // Adds result only if x is valid. Then, if value is invalid, it is set to NaN
120 if (!std::isnan(x)) {
121 if (!std::isnan(x)) {
121 xData.push_back(x);
122 xData.push_back(x);
122
123
123 // Values
124 // Values
124 for (auto valueIndex = 0; valueIndex < expectedNbValues; ++valueIndex) {
125 for (auto valueIndex = 1; valueIndex < expectedNbValues; ++valueIndex) {
125 auto column = valueIndex + 1;
126 auto column = valueIndex;
126
127
127 bool valueOk;
128 bool valueOk;
128 auto value = lineData.at(column).toDouble(&valueOk);
129 auto value = lineData.at(column).toDouble(&valueOk);
129
130
130 if (!valueOk) {
131 if (!valueOk) {
131 qCWarning(LOG_AmdaResultParser())
132 qCWarning(LOG_AmdaResultParser())
132 << QObject::tr(
133 << QObject::tr(
133 "Value from (line %1, column %2) is invalid and will be "
134 "Value from (line %1, column %2) is invalid and will be "
134 "converted to NaN")
135 "converted to NaN")
135 .arg(line, column);
136 .arg(line, column);
136 value = std::numeric_limits<double>::quiet_NaN();
137 value = std::numeric_limits<double>::quiet_NaN();
137 }
138 }
138 valuesData.push_back(value);
139 valuesData.push_back(value);
139 }
140 }
140 }
141 }
141 else {
142 else {
142 qCWarning(LOG_AmdaResultParser())
143 qCWarning(LOG_AmdaResultParser())
143 << QObject::tr("Can't retrieve results from line %1: x is invalid")
144 << QObject::tr("Can't retrieve results from line %1: x is invalid")
144 .arg(line);
145 .arg(line);
145 }
146 }
146 }
147 }
147 else {
148 else {
148 qCWarning(LOG_AmdaResultParser())
149 qCWarning(LOG_AmdaResultParser())
149 << QObject::tr("Can't retrieve results from line %1: invalid line").arg(line);
150 << QObject::tr("Can't retrieve results from line %1: invalid line").arg(line);
150 }
151 }
151 }
152 } while (stream.readLineInto(&line));
152 }
153 }
153
154
154 return std::make_pair(std::move(xData), std::move(valuesData));
155 return std::make_pair(std::move(xData), std::move(valuesData));
155 }
156 }
156
157
157 } // namespace
158 } // namespace
158
159
159 std::shared_ptr<IDataSeries> AmdaResultParser::readTxt(const QString &filePath,
160 std::shared_ptr<IDataSeries> AmdaResultParser::readTxt(const QString &filePath,
160 ValueType valueType) noexcept
161 ValueType valueType) noexcept
161 {
162 {
162 if (valueType == ValueType::UNKNOWN) {
163 if (valueType == ValueType::UNKNOWN) {
163 qCCritical(LOG_AmdaResultParser())
164 qCCritical(LOG_AmdaResultParser())
164 << QObject::tr("Can't retrieve AMDA data: the type of values to be read is unknown");
165 << QObject::tr("Can't retrieve AMDA data: the type of values to be read is unknown");
165 return nullptr;
166 return nullptr;
166 }
167 }
167
168
168 QFile file{filePath};
169 QFile file{filePath};
169
170
170 if (!file.open(QFile::ReadOnly | QIODevice::Text)) {
171 if (!file.open(QFile::ReadOnly | QIODevice::Text)) {
171 qCCritical(LOG_AmdaResultParser())
172 qCCritical(LOG_AmdaResultParser())
172 << QObject::tr("Can't retrieve AMDA data from file %1: %2")
173 << QObject::tr("Can't retrieve AMDA data from file %1: %2")
173 .arg(filePath, file.errorString());
174 .arg(filePath, file.errorString());
174 return nullptr;
175 return nullptr;
175 }
176 }
176
177
177 QTextStream stream{&file};
178 QTextStream stream{&file};
178
179
179 // Checks if the file was found on the server
180 // Checks if the file was found on the server
180 auto firstLine = stream.readLine();
181 auto firstLine = stream.readLine();
181 if (firstLine.compare(FILE_NOT_FOUND_MESSAGE) == 0) {
182 if (firstLine.compare(FILE_NOT_FOUND_MESSAGE) == 0) {
182 qCCritical(LOG_AmdaResultParser())
183 qCCritical(LOG_AmdaResultParser())
183 << QObject::tr("Can't retrieve AMDA data from file %1: file was not found on server")
184 << QObject::tr("Can't retrieve AMDA data from file %1: file was not found on server")
184 .arg(filePath);
185 .arg(filePath);
185 return nullptr;
186 return nullptr;
186 }
187 }
187
188
188 // Reads x-axis unit
189 // Reads x-axis unit
189 stream.seek(0); // returns to the beginning of the file
190 stream.seek(0); // returns to the beginning of the file
190 auto xAxisUnit = readXAxisUnit(stream);
191 auto xAxisUnit = readXAxisUnit(stream);
191
192
192 // Reads results
193 // Reads results
193 auto results = readResults(stream, valueType);
194 auto results = readResults(stream, valueType);
194
195
195 // Creates data series
196 // Creates data series
196 switch (valueType) {
197 switch (valueType) {
197 case ValueType::SCALAR:
198 case ValueType::SCALAR:
198 return std::make_shared<ScalarSeries>(std::move(results.first),
199 return std::make_shared<ScalarSeries>(std::move(results.first),
199 std::move(results.second), xAxisUnit, Unit{});
200 std::move(results.second), xAxisUnit, Unit{});
200 case ValueType::VECTOR:
201 case ValueType::VECTOR:
201 return std::make_shared<VectorSeries>(std::move(results.first),
202 return std::make_shared<VectorSeries>(std::move(results.first),
202 std::move(results.second), xAxisUnit, Unit{});
203 std::move(results.second), xAxisUnit, Unit{});
203 case ValueType::UNKNOWN:
204 case ValueType::UNKNOWN:
204 // Invalid case
205 // Invalid case
205 break;
206 break;
206 }
207 }
207
208
208 // Invalid cases
209 // Invalid cases
209 qCCritical(LOG_AmdaResultParser())
210 qCCritical(LOG_AmdaResultParser())
210 << QObject::tr("Can't create data series: unsupported value type");
211 << QObject::tr("Can't create data series: unsupported value type");
211 return nullptr;
212 return nullptr;
212 }
213 }
General Comments 0
You need to be logged in to leave comments. Login now