From bd6284620dee84a53c35b49d35256ca8a93349f0 2017-11-23 16:47:02 From: Alexandre Leroux Date: 2017-11-23 16:47:02 Subject: [PATCH] Merge branch 'feature/SpectroMeshes' into develop --- diff --git a/core/include/Data/DataSeries.h b/core/include/Data/DataSeries.h index c258397..352896d 100644 --- a/core/include/Data/DataSeries.h +++ b/core/include/Data/DataSeries.h @@ -36,7 +36,9 @@ public: template > explicit IteratorValue(DataSeries &dataSeries, bool begin) : m_XIt(begin ? dataSeries.xAxisData()->begin() : dataSeries.xAxisData()->end()), - m_ValuesIt(begin ? dataSeries.valuesData()->begin() : dataSeries.valuesData()->end()) + m_ValuesIt(begin ? dataSeries.valuesData()->begin() : dataSeries.valuesData()->end()), + m_YItBegin{dataSeries.yAxis().begin()}, + m_YItEnd{dataSeries.yAxis().end()} { } @@ -44,7 +46,9 @@ public: explicit IteratorValue(const DataSeries &dataSeries, bool begin) : m_XIt(begin ? dataSeries.xAxisData()->cbegin() : dataSeries.xAxisData()->cend()), m_ValuesIt(begin ? dataSeries.valuesData()->cbegin() - : dataSeries.valuesData()->cend()) + : dataSeries.valuesData()->cend()), + m_YItBegin{dataSeries.yAxis().cbegin()}, + m_YItEnd{dataSeries.yAxis().cend()} { } @@ -65,7 +69,9 @@ public: bool equals(const DataSeriesIteratorValue::Impl &other) const override try { const auto &otherImpl = dynamic_cast(other); - return std::tie(m_XIt, m_ValuesIt) == std::tie(otherImpl.m_XIt, otherImpl.m_ValuesIt); + return std::tie(m_XIt, m_ValuesIt, m_YItBegin, m_YItEnd) + == std::tie(otherImpl.m_XIt, otherImpl.m_ValuesIt, otherImpl.m_YItBegin, + otherImpl.m_YItEnd); } catch (const std::bad_cast &) { return false; @@ -99,6 +105,15 @@ public: } double x() const override { return m_XIt->at(0); } + std::vector y() const override + { + std::vector result{}; + std::transform(m_YItBegin, m_YItEnd, std::back_inserter(result), + [](const auto &it) { return it.first(); }); + + return result; + } + double value() const override { return m_ValuesIt->at(0); } double value(int componentIndex) const override { return m_ValuesIt->at(componentIndex); } double minValue() const override { return m_ValuesIt->min(); } @@ -110,11 +125,15 @@ public: auto &otherImpl = dynamic_cast(other); m_XIt->impl()->swap(*otherImpl.m_XIt->impl()); m_ValuesIt->impl()->swap(*otherImpl.m_ValuesIt->impl()); + m_YItBegin->impl()->swap(*otherImpl.m_YItBegin->impl()); + m_YItEnd->impl()->swap(*otherImpl.m_YItEnd->impl()); } private: ArrayDataIterator m_XIt; ArrayDataIterator m_ValuesIt; + ArrayDataIterator m_YItBegin; + ArrayDataIterator m_YItEnd; }; } // namespace dataseries_detail @@ -193,6 +212,9 @@ public: /// @sa IDataSeries::xAxisUnit() Unit xAxisUnit() const override { return m_XAxisUnit; } + /// @sa IDataSeries::yAxisUnit() + Unit yAxisUnit() const override { return m_YAxis.unit(); } + /// @return the values dataset std::shared_ptr > valuesData() { return m_ValuesData; } const std::shared_ptr > valuesData() const { return m_ValuesData; } @@ -202,6 +224,8 @@ public: int nbPoints() const override { return m_ValuesData->totalSize(); } + std::pair yBounds() const override { return m_YAxis.bounds(); } + void clear() { m_XAxisData->clear(); @@ -384,8 +408,8 @@ public: } /// @return the y-axis associated to the data series - /// @todo pass getter as protected and use iterators to access the y-axis data - OptionalAxis yAxis() const { return m_YAxis; } + const OptionalAxis &yAxis() const { return m_YAxis; } + OptionalAxis &yAxis() { return m_YAxis; } // /////// // // Mutexes // diff --git a/core/include/Data/DataSeriesIterator.h b/core/include/Data/DataSeriesIterator.h index a6b1f05..92c251d 100644 --- a/core/include/Data/DataSeriesIterator.h +++ b/core/include/Data/DataSeriesIterator.h @@ -27,6 +27,7 @@ public: virtual void next(int offset) = 0; virtual void prev() = 0; virtual double x() const = 0; + virtual std::vector y() const = 0; virtual double value() const = 0; virtual double value(int componentIndex) const = 0; virtual double minValue() const = 0; @@ -51,6 +52,8 @@ public: void prev(); /// Gets x-axis data double x() const; + /// Gets y-axis data + std::vector y() const; /// Gets value data double value() const; /// Gets value data depending on an index diff --git a/core/include/Data/DataSeriesUtils.h b/core/include/Data/DataSeriesUtils.h new file mode 100644 index 0000000..e696345 --- /dev/null +++ b/core/include/Data/DataSeriesUtils.h @@ -0,0 +1,228 @@ +#ifndef SCIQLOP_DATASERIESUTILS_H +#define SCIQLOP_DATASERIESUTILS_H + +#include "CoreGlobal.h" + +#include + +#include +#include + +Q_DECLARE_LOGGING_CATEGORY(LOG_DataSeriesUtils) + +/** + * Utility class with methods for data series + */ +struct SCIQLOP_CORE_EXPORT DataSeriesUtils { + /** + * Define a meshs. + * + * A mesh is a regular grid representing cells of the same width (in x) and of the same height + * (in y). At each mesh point is associated a value. + * + * Each axis of the mesh is defined by a minimum value, a number of values is a mesh step. + * For example: if min = 1, nbValues = 5 and step = 2 => the axis of the mesh will be [1, 3, 5, + * 7, 9]. + * + * The values are defined in an array of size {nbX * nbY}. The data is stored along the X axis. + * + * For example, the mesh: + * Y = 2 [ 7 ; 8 ; 9 + * Y = 1 4 ; 5 ; 6 + * Y = 0 1 ; 2 ; 3 ] + * X = 0 X = 1 X = 2 + * + * will be represented by data [1, 2, 3, 4, 5, 6, 7, 8, 9] + */ + struct Mesh { + explicit Mesh() = default; + explicit Mesh(int nbX, double xMin, double xStep, int nbY, double yMin, double yStep) + : m_NbX{nbX}, + m_XMin{xMin}, + m_XStep{xStep}, + m_NbY{nbY}, + m_YMin{yMin}, + m_YStep{yStep}, + m_Data(nbX * nbY) + { + } + + inline bool isEmpty() const { return m_Data.size() == 0; } + inline double xMax() const { return m_XMin + (m_NbX - 1) * m_XStep; } + inline double yMax() const { return m_YMin + (m_NbY - 1) * m_YStep; } + + int m_NbX{0}; + double m_XMin{}; + double m_XStep{}; + int m_NbY{0}; + double m_YMin{}; + double m_YStep{}; + std::vector m_Data{}; + }; + + /** + * Represents a resolution used to generate the data of a mesh on the x-axis or in Y. + * + * A resolution is represented by a value and flag indicating if it's in the logarithmic scale + * @sa Mesh + */ + struct Resolution { + double m_Val{std::numeric_limits::quiet_NaN()}; + bool m_Logarithmic{false}; + }; + + /** + * Processes data from a data series to complete the data holes with a fill value. + * + * A data hole is determined by the resolution passed in parameter: if, between two continuous + * data on the x-axis, the difference between these data is greater than the resolution, then + * there is one or more holes between them. The holes are filled by adding: + * - for the x-axis, new data corresponding to the 'step resolution' starting from the first + * data; + * - for values, a default value (fill value) for each new data added on the x-axis. + * + * For example, with : + * - xAxisData = [0, 1, 5, 7, 14 ] + * - valuesData = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] (two components per x-axis data) + * - fillValue = NaN + * - and resolution = 2; + * + * For the x axis, we calculate as data holes: [3, 9, 11, 13]. These holes are added to the + * x-axis data, and NaNs (two per x-axis data) are added to the values: + * => xAxisData = [0, 1, 3, 5, 7, 9, 11, 13, 14 ] + * => valuesData = [0, 1, 2, 3, NaN, NaN, 4, 5, 6, 7, NaN, NaN, NaN, NaN, NaN, NaN, 8, 9] + * + * It is also possible to set bounds for the data series. If these bounds are defined and exceed + * the limits of the data series, data holes are added to the series at the beginning and/or the + * end. + * + * The generation of data holes at the beginning/end of the data series is performed starting + * from the x-axis series limit and adding data holes at each 'resolution step' as long as the + * new bound is not reached. + * + * For example, with : + * - xAxisData = [3, 4, 5, 6, 7 ] + * - valuesData = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + * - fillValue = NaN + * - minBound = 0 + * - maxBound = 12 + * - and resolution = 2; + * + * => Starting from 3 and decreasing 2 by 2 until reaching 0 : a data hole at value 1 will be + * added to the beginning of the series + * => Starting from 7 and increasing 2 by 2 until reaching 12 : data holes at values 9 and 11 + * will be added to the end of the series + * + * So : + * => xAxisData = [1, 3, 4, 5, 6, 7, 9, 11 ] + * => valuesData = [NaN, NaN, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, NaN, NaN, NaN, NaN] + * + * @param xAxisData the x-axis data of the data series + * @param valuesData the values data of the data series + * @param resolution the resoultion (on x-axis) used to determinate data holes + * @param fillValue the fill value used for data holes in the values data + * @param minBound the limit at which to start filling data holes for the series. If set to NaN, + * the limit is not used + * @param maxBound the limit at which to end filling data holes for the series. If set to NaN, + * the limit is not used + * + * @remarks There is no control over the consistency between x-axis data and values data. The + * method considers that the data is well formed (the total number of values data is a multiple + * of the number of x-axis data) + */ + static void fillDataHoles(std::vector &xAxisData, std::vector &valuesData, + double resolution, + double fillValue = std::numeric_limits::quiet_NaN(), + double minBound = std::numeric_limits::quiet_NaN(), + double maxBound = std::numeric_limits::quiet_NaN()); + /** + * Computes the resolution of a dataset passed as a parameter. + * + * The resolution of a dataset is the minimum difference between two values that follow in the + * set. + * For example: + * - for the set [0, 2, 4, 8, 10, 11, 13] => the resolution is 1 (difference between 10 and 11). + * + * A resolution can be calculated on the logarithmic scale (base of 10). In this case, the + * dataset is first converted to logarithmic values. + * For example: + * - for the set [10, 100, 10000, 1000000], the values are converted to [1, 2, 4, 6] => the + * logarithmic resolution is 1 (difference between 1 and 2). + * + * @param begin the iterator pointing to the beginning of the dataset + * @param end the iterator pointing to the end of the dataset + * @param logarithmic computes a logarithmic resolution or not + * @return the resolution computed + * @warning the method considers the dataset as sorted and doesn't control it. + */ + template + static Resolution resolution(Iterator begin, Iterator end, bool logarithmic = false); + + /** + * Computes a regular mesh for a data series, according to resolutions for x-axis and y-axis + * passed as parameters. + * + * The mesh is created from the resolutions in x and y and the boundaries delimiting the data + * series. If the resolutions do not allow to obtain a regular mesh, they are recalculated. + * + * For example : + * Let x-axis data = [0, 1, 3, 5, 9], its associated values ​​= [0, 10, 30, 50, 90] and + * xResolution = 2. + * Based on the resolution, the mesh would be [0, 2, 4, 6, 8, 10] and would be invalid because + * it exceeds the maximum bound of the data. The resolution is thus recalculated so that the + * mesh holds between the data terminals. + * So => resolution is 1.8 and the mesh is [0, 1.8, 3.6, 5.4, 7.2, 9]. + * + * Once the mesh is generated in x and y, the values ​​are associated with each mesh point, + * based on the data in the series, finding the existing data at which the mesh point would be + * or would be closest to, without exceeding it. + * + * In the example, we determine the value of each mesh point: + * - x = 0 => value = 0 (existing x in the data series) + * - x = 1.8 => value = 10 (the closest existing x: 1) + * - x = 3.6 => value = 30 (the closest existing x: 3) + * - x = 5.4 => value = 50 (the closest existing x: 5) + * - x = 7.2 => value = 50 (the closest existing x: 5) + * - x = 9 => value = 90 (existing x in the data series) + * + * Same algorithm is applied for y-axis. + * + * @param begin the iterator pointing to the beginning of the data series + * @param end the iterator pointing to the end of the data series + * @param xResolution the resolution expected for the mesh's x-axis + * @param yResolution the resolution expected for the mesh's y-axis + * @return the mesh created, an empty mesh if the input data do not allow to generate a regular + * mesh (empty data, null resolutions, logarithmic x-axis) + * @warning the method considers the dataset as sorted and doesn't control it. + */ + static Mesh regularMesh(DataSeriesIterator begin, DataSeriesIterator end, + Resolution xResolution, Resolution yResolution); +}; + +template +DataSeriesUtils::Resolution DataSeriesUtils::resolution(Iterator begin, Iterator end, + bool logarithmic) +{ + // Retrieves data into a work dataset + using ValueType = typename Iterator::value_type; + std::vector values{}; + std::copy(begin, end, std::back_inserter(values)); + + // Converts data if logarithmic flag is activated + if (logarithmic) { + std::for_each(values.begin(), values.end(), + [logarithmic](auto &val) { val = std::log10(val); }); + } + + // Computes the differences between the values in the dataset + std::adjacent_difference(values.begin(), values.end(), values.begin()); + + // Retrieves the smallest difference + auto resolutionIt = std::min_element(values.begin(), values.end()); + auto resolution + = resolutionIt != values.end() ? *resolutionIt : std::numeric_limits::quiet_NaN(); + + return Resolution{resolution, logarithmic}; +} + +#endif // SCIQLOP_DATASERIESUTILS_H diff --git a/core/include/Data/IDataSeries.h b/core/include/Data/IDataSeries.h index 44ed1dd..c16f6c4 100644 --- a/core/include/Data/IDataSeries.h +++ b/core/include/Data/IDataSeries.h @@ -39,6 +39,9 @@ public: virtual Unit xAxisUnit() const = 0; + /// @return the y-axis unit, if axis is defined, default unit otherwise + virtual Unit yAxisUnit() const = 0; + virtual Unit valuesUnit() const = 0; virtual void merge(IDataSeries *dataSeries) = 0; @@ -53,6 +56,9 @@ public: /// @return the total number of points contained in the data series virtual int nbPoints() const = 0; + /// @return the bounds of the y-axis axis (if defined) + virtual std::pair yBounds() const = 0; + // ///////// // // Iterators // // ///////// // diff --git a/core/include/Data/OptionalAxis.h b/core/include/Data/OptionalAxis.h index 73e6ae2..f3a93c9 100644 --- a/core/include/Data/OptionalAxis.h +++ b/core/include/Data/OptionalAxis.h @@ -1,6 +1,8 @@ #ifndef SCIQLOP_OPTIONALAXIS_H #define SCIQLOP_OPTIONALAXIS_H +#include + #include "CoreGlobal.h" #include "Unit.h" @@ -38,10 +40,6 @@ public: /// @return the flag that indicates if the axis is defined or not bool isDefined() const; - /// @return gets the data at the index passed in parameter, NaN if the index is outside the - /// bounds of the axis, or if the axis is undefined - double at(int index) const; - ///@return the min and max values of the data on the axis, NaN values if there is no data std::pair bounds() const; @@ -53,6 +51,12 @@ public: bool operator==(const OptionalAxis &other); bool operator!=(const OptionalAxis &other); + // Iterators on data + ArrayDataIterator begin(); + ArrayDataIterator end(); + ArrayDataIterator cbegin() const; + ArrayDataIterator cend() const; + private: bool m_Defined; ///< Axis is defined or not std::shared_ptr > m_Data; ///< Axis' data diff --git a/core/include/Data/SpectrogramSeries.h b/core/include/Data/SpectrogramSeries.h index a58bc74..f13ea1e 100644 --- a/core/include/Data/SpectrogramSeries.h +++ b/core/include/Data/SpectrogramSeries.h @@ -16,18 +16,25 @@ public: /// Ctor explicit SpectrogramSeries(std::vector xAxisData, std::vector yAxisData, std::vector valuesData, const Unit &xAxisUnit, - const Unit &yAxisUnit, const Unit &valuesUnit); + const Unit &yAxisUnit, const Unit &valuesUnit, + double xResolution = std::numeric_limits::quiet_NaN()); /// Ctor directly with the y-axis explicit SpectrogramSeries(std::shared_ptr > xAxisData, const Unit &xAxisUnit, std::shared_ptr > valuesData, const Unit &valuesUnit, - OptionalAxis yAxis); + OptionalAxis yAxis, + double xResolution = std::numeric_limits::quiet_NaN()); /// @sa DataSeries::clone() std::unique_ptr clone() const override; /// @sa DataSeries::subDataSeries() std::shared_ptr subDataSeries(const SqpRange &range) override; + + inline double xResolution() const noexcept { return m_XResolution; } + +private: + double m_XResolution; ///< Resolution used on x-axis to build the spectrogram }; #endif // SCIQLOP_SPECTROGRAMSERIES_H diff --git a/core/meson.build b/core/meson.build index 2a527f6..11a1fb6 100644 --- a/core/meson.build +++ b/core/meson.build @@ -27,6 +27,7 @@ core_sources = [ 'src/Data/ArrayDataIterator.cpp', 'src/Data/VectorSeries.cpp', 'src/Data/OptionalAxis.cpp', + 'src/Data/DataSeriesUtils.cpp', 'src/DataSource/DataSourceController.cpp', 'src/DataSource/DataSourceItem.cpp', 'src/DataSource/DataSourceItemAction.cpp', diff --git a/core/src/Data/DataSeriesIterator.cpp b/core/src/Data/DataSeriesIterator.cpp index 5066fe7..8a3dd13 100644 --- a/core/src/Data/DataSeriesIterator.cpp +++ b/core/src/Data/DataSeriesIterator.cpp @@ -52,6 +52,11 @@ double DataSeriesIteratorValue::x() const return m_Impl->x(); } +std::vector DataSeriesIteratorValue::y() const +{ + return m_Impl->y(); +} + double DataSeriesIteratorValue::value() const { return m_Impl->value(); diff --git a/core/src/Data/DataSeriesUtils.cpp b/core/src/Data/DataSeriesUtils.cpp new file mode 100644 index 0000000..a4e4120 --- /dev/null +++ b/core/src/Data/DataSeriesUtils.cpp @@ -0,0 +1,194 @@ +#include "Data/DataSeriesUtils.h" + +Q_LOGGING_CATEGORY(LOG_DataSeriesUtils, "DataSeriesUtils") + +void DataSeriesUtils::fillDataHoles(std::vector &xAxisData, std::vector &valuesData, + double resolution, double fillValue, double minBound, + double maxBound) +{ + if (resolution == 0. || std::isnan(resolution)) { + qCWarning(LOG_DataSeriesUtils()) + << "Can't fill data holes with a null resolution, no changes will be made"; + return; + } + + if (xAxisData.empty()) { + qCWarning(LOG_DataSeriesUtils()) + << "Can't fill data holes for empty data, no changes will be made"; + return; + } + + // Gets the number of values per x-axis data + auto nbComponents = valuesData.size() / xAxisData.size(); + + // Generates fill values that will be used to complete values data + std::vector fillValues(nbComponents, fillValue); + + // Checks if there are data holes on the beginning of the data and generates the hole at the + // extremity if it's the case + auto minXAxisData = xAxisData.front(); + if (!std::isnan(minBound) && minBound < minXAxisData) { + auto holeSize = static_cast((minXAxisData - minBound) / resolution); + if (holeSize > 0) { + xAxisData.insert(xAxisData.begin(), minXAxisData - holeSize * resolution); + valuesData.insert(valuesData.begin(), fillValues.begin(), fillValues.end()); + } + } + + // Same for the end of the data + auto maxXAxisData = xAxisData.back(); + if (!std::isnan(maxBound) && maxBound > maxXAxisData) { + auto holeSize = static_cast((maxBound - maxXAxisData) / resolution); + if (holeSize > 0) { + xAxisData.insert(xAxisData.end(), maxXAxisData + holeSize * resolution); + valuesData.insert(valuesData.end(), fillValues.begin(), fillValues.end()); + } + } + + // Generates other data holes + auto xAxisIt = xAxisData.begin(); + while (xAxisIt != xAxisData.end()) { + // Stops at first value which has a gap greater than resolution with the value next to it + xAxisIt = std::adjacent_find( + xAxisIt, xAxisData.end(), + [resolution](const auto &a, const auto &b) { return (b - a) > resolution; }); + + if (xAxisIt != xAxisData.end()) { + auto nextXAxisIt = xAxisIt + 1; + + // Gets the values that has a gap greater than resolution between them + auto lowValue = *xAxisIt; + auto highValue = *nextXAxisIt; + + // Completes holes between the two values by creating new values (according to the + // resolution) + for (auto i = lowValue + resolution; i < highValue; i += resolution) { + // Gets the iterator of values data from which to insert fill values + auto nextValuesIt = valuesData.begin() + + std::distance(xAxisData.begin(), nextXAxisIt) * nbComponents; + + // New value is inserted before nextXAxisIt + nextXAxisIt = xAxisData.insert(nextXAxisIt, i) + 1; + + // New values are inserted before nextValuesIt + valuesData.insert(nextValuesIt, fillValues.begin(), fillValues.end()); + } + + // Moves to the next value to continue loop on the x-axis data + xAxisIt = nextXAxisIt; + } + } +} + +namespace { + +/** + * Generates axis's mesh properties according to data and resolution + * @param begin the iterator pointing to the beginning of the data + * @param end the iterator pointing to the end of the data + * @param fun the function to retrieve data from the data iterators + * @param resolution the resolution to use for the axis' mesh + * @return a tuple representing the mesh properties : + */ +template +std::tuple meshProperties(Iterator begin, Iterator end, IteratorFun fun, + double resolution) +{ + // Computes the gap between min and max data. This will be used to determinate the step between + // each data of the mesh + auto min = fun(begin); + auto max = fun(end - 1); + auto gap = max - min; + + // Computes the step trying to use the fixed resolution. If the resolution doesn't separate the + // values evenly , it is recalculated. + // For example, for a resolution of 2.0: + // - for interval [0; 8] => resolution is valid, the generated mesh will be [0, 2, 4, 6, 8] + // - for interval [0; 9] => it's impossible to create a regular mesh with this resolution + // The resolution is recalculated and is worth 1.8. The generated mesh will be [0, 1.8, 3.6, + // 5.4, 7.2, 9] + auto nbVal = static_cast(std::ceil(gap / resolution)); + auto step = gap / nbVal; + + // last data is included in the total number of values + return std::make_tuple(nbVal + 1, min, step); +} + +} // namespace + +DataSeriesUtils::Mesh DataSeriesUtils::regularMesh(DataSeriesIterator begin, DataSeriesIterator end, + Resolution xResolution, Resolution yResolution) +{ + // Checks preconditions + if (xResolution.m_Val == 0. || std::isnan(xResolution.m_Val) || yResolution.m_Val == 0. + || std::isnan(yResolution.m_Val)) { + qCWarning(LOG_DataSeriesUtils()) << "Can't generate mesh with a null resolution"; + return Mesh{}; + } + + if (xResolution.m_Logarithmic) { + qCWarning(LOG_DataSeriesUtils()) + << "Can't generate mesh with a logarithmic x-axis resolution"; + return Mesh{}; + } + + if (std::distance(begin, end) == 0) { + qCWarning(LOG_DataSeriesUtils()) << "Can't generate mesh for empty data"; + return Mesh{}; + } + + auto yData = begin->y(); + if (yData.empty()) { + qCWarning(LOG_DataSeriesUtils()) << "Can't generate mesh for data with no y-axis"; + return Mesh{}; + } + + // Converts y-axis and its resolution to logarithmic values + if (yResolution.m_Logarithmic) { + std::for_each(yData.begin(), yData.end(), [](auto &val) { val = std::log10(val); }); + } + + // Computes mesh properties + int nbX, nbY; + double xMin, xStep, yMin, yStep; + std::tie(nbX, xMin, xStep) + = meshProperties(begin, end, [](const auto &it) { return it->x(); }, xResolution.m_Val); + std::tie(nbY, yMin, yStep) = meshProperties( + yData.begin(), yData.end(), [](const auto &it) { return *it; }, yResolution.m_Val); + + // Generates mesh according to the x-axis and y-axis steps + Mesh result{nbX, xMin, xStep, nbY, yMin, yStep}; + + for (auto meshXIndex = 0; meshXIndex < nbX; ++meshXIndex) { + auto meshX = xMin + meshXIndex * xStep; + // According to current x-axis of the mesh, finds in the data series the interval in which + // the data is or gets closer (without exceeding it). + // An interval is defined by a value and extends to +/- 50% of the resolution. For example, + // for a value of 3 and a resolution of 1, the associated interval is [2.5, 3.5]. + auto xIt = std::lower_bound(begin, end, meshX, + [xResolution](const auto &it, const auto &val) { + return it.x() - xResolution.m_Val / 2. < val; + }) + - 1; + + // When the corresponding entry of the data series is found, generates the values of the + // mesh by retrieving the values of the entry, for each y-axis value of the mesh + auto values = xIt->values(); + + for (auto meshYIndex = 0; meshYIndex < nbY; ++meshYIndex) { + auto meshY = yMin + meshYIndex * yStep; + + auto yBegin = yData.begin(); + auto yIt = std::lower_bound(yBegin, yData.end(), meshY, + [yResolution](const auto &it, const auto &val) { + return it - yResolution.m_Val / 2. < val; + }) + - 1; + + auto valueIndex = std::distance(yBegin, yIt); + result.m_Data[result.m_NbX * meshYIndex + meshXIndex] = values.at(valueIndex); + } + } + + return result; +} diff --git a/core/src/Data/OptionalAxis.cpp b/core/src/Data/OptionalAxis.cpp index e085bea..30004f4 100644 --- a/core/src/Data/OptionalAxis.cpp +++ b/core/src/Data/OptionalAxis.cpp @@ -2,7 +2,8 @@ #include "Data/ArrayData.h" -OptionalAxis::OptionalAxis() : m_Defined{false}, m_Data{nullptr}, m_Unit{} +OptionalAxis::OptionalAxis() + : m_Defined{false}, m_Data{std::make_shared >(std::vector{})}, m_Unit{} { } @@ -15,9 +16,7 @@ OptionalAxis::OptionalAxis(std::shared_ptr > data, Unit unit) } OptionalAxis::OptionalAxis(const OptionalAxis &other) - : m_Defined{other.m_Defined}, - m_Data{other.m_Data ? std::make_shared >(*other.m_Data) : nullptr}, - m_Unit{other.m_Unit} + : m_Defined{other.m_Defined}, m_Data{other.m_Data}, m_Unit{other.m_Unit} { } @@ -35,17 +34,6 @@ bool OptionalAxis::isDefined() const return m_Defined; } -double OptionalAxis::at(int index) const -{ - if (m_Defined) { - return (index >= 0 && index < m_Data->size()) ? m_Data->at(index) - : std::numeric_limits::quiet_NaN(); - } - else { - return std::numeric_limits::quiet_NaN(); - } -} - std::pair OptionalAxis::bounds() const { if (!m_Defined || m_Data->size() == 0) { @@ -97,3 +85,23 @@ bool OptionalAxis::operator!=(const OptionalAxis &other) { return !(*this == other); } + +ArrayDataIterator OptionalAxis::begin() +{ + return m_Data->begin(); +} + +ArrayDataIterator OptionalAxis::end() +{ + return m_Data->end(); +} + +ArrayDataIterator OptionalAxis::cbegin() const +{ + return m_Data->cbegin(); +} + +ArrayDataIterator OptionalAxis::cend() const +{ + return m_Data->cend(); +} diff --git a/core/src/Data/SpectrogramSeries.cpp b/core/src/Data/SpectrogramSeries.cpp index 2d91860..2ea57ba 100644 --- a/core/src/Data/SpectrogramSeries.cpp +++ b/core/src/Data/SpectrogramSeries.cpp @@ -2,20 +2,25 @@ SpectrogramSeries::SpectrogramSeries(std::vector xAxisData, std::vector yAxisData, std::vector valuesData, const Unit &xAxisUnit, - const Unit &yAxisUnit, const Unit &valuesUnit) + const Unit &yAxisUnit, const Unit &valuesUnit, + double resolution) : SpectrogramSeries{ - std::make_shared >(std::move(xAxisData)), xAxisUnit, - std::make_shared >(std::move(valuesData), yAxisData.size()), valuesUnit, - OptionalAxis{std::make_shared >(std::move(yAxisData)), yAxisUnit}} + std::make_shared >(std::move(xAxisData)), + xAxisUnit, + std::make_shared >(std::move(valuesData), yAxisData.size()), + valuesUnit, + OptionalAxis{std::make_shared >(std::move(yAxisData)), yAxisUnit}, + resolution} { } SpectrogramSeries::SpectrogramSeries(std::shared_ptr > xAxisData, const Unit &xAxisUnit, std::shared_ptr > valuesData, - const Unit &valuesUnit, OptionalAxis yAxis) + const Unit &valuesUnit, OptionalAxis yAxis, double resolution) : DataSeries{std::move(xAxisData), xAxisUnit, std::move(valuesData), valuesUnit, - std::move(yAxis)} + std::move(yAxis)}, + m_XResolution{resolution} { } diff --git a/core/tests/Data/DataSeriesUtils.cpp b/core/tests/Data/DataSeriesTestsUtils.cpp similarity index 96% rename from core/tests/Data/DataSeriesUtils.cpp rename to core/tests/Data/DataSeriesTestsUtils.cpp index 25abb3e..70ec818 100644 --- a/core/tests/Data/DataSeriesUtils.cpp +++ b/core/tests/Data/DataSeriesTestsUtils.cpp @@ -1,4 +1,4 @@ -#include "DataSeriesUtils.h" +#include "DataSeriesTestsUtils.h" void validateRange(DataSeriesIterator first, DataSeriesIterator last, const DataContainer &xData, const DataContainer &valuesData) diff --git a/core/tests/Data/DataSeriesUtils.h b/core/tests/Data/DataSeriesTestsUtils.h similarity index 96% rename from core/tests/Data/DataSeriesUtils.h rename to core/tests/Data/DataSeriesTestsUtils.h index d12657f..edcb5d6 100644 --- a/core/tests/Data/DataSeriesUtils.h +++ b/core/tests/Data/DataSeriesTestsUtils.h @@ -1,12 +1,12 @@ /** - * The DataSeriesUtils file contains a set of utility methods that can be used to test the operations on a DataSeries. + * The DataSeriesTestsUtils file contains a set of utility methods that can be used to test the operations on a DataSeries. * * Most of these methods are template methods to adapt to any series (scalars, vectors, spectrograms...) * * @sa DataSeries */ -#ifndef SCIQLOP_DATASERIESUTILS_H -#define SCIQLOP_DATASERIESUTILS_H +#ifndef SCIQLOP_DATASERIESTESTSUTILS_H +#define SCIQLOP_DATASERIESTESTSUTILS_H #include #include @@ -370,4 +370,4 @@ void testValuesBounds_t() } } -#endif // SCIQLOP_DATASERIESUTILS_H +#endif // SCIQLOP_DATASERIESTESTSUTILS_H diff --git a/core/tests/Data/TestDataSeriesUtils.cpp b/core/tests/Data/TestDataSeriesUtils.cpp new file mode 100644 index 0000000..a0854fa --- /dev/null +++ b/core/tests/Data/TestDataSeriesUtils.cpp @@ -0,0 +1,108 @@ +#include + +#include +#include + +class TestDataSeriesUtils : public QObject { + Q_OBJECT + +private slots: + /// Tests @sa DataSeriesUtils::fillDataHoles() method + void testFillDataHoles_data(); + void testFillDataHoles(); +}; + +void TestDataSeriesUtils::testFillDataHoles_data() +{ + QTest::addColumn >("xAxisData"); + QTest::addColumn >("valuesData"); + QTest::addColumn("resolution"); + QTest::addColumn("fillValue"); + QTest::addColumn("minBound"); + QTest::addColumn("maxBound"); + QTest::addColumn >( + "expectedXAxisData"); // expected x-axis data after filling holes + QTest::addColumn >( + "expectedValuesData"); // expected values data after filling holes + + auto nan = std::numeric_limits::quiet_NaN(); + + QTest::newRow("fillDataHoles (basic case)") + << std::vector{0., 1., 5., 7., 14.} << std::vector{0., 1., 2., 3., 4.} << 2. + << nan << nan << nan << std::vector{0., 1., 3., 5., 7., 9., 11., 13., 14.} + << std::vector{0., 1., nan, 2., 3., nan, nan, nan, 4.}; + + QTest::newRow("fillDataHoles (change nb components)") + << std::vector{0., 1., 5., 7., 14.} + << std::vector{0., 1., 2., 3., 4., 5., 6., 7., 8., 9.} << 2. << nan << nan << nan + << std::vector{0., 1., 3., 5., 7., 9., 11., 13., 14.} + << std::vector{0., 1., 2., 3., nan, nan, 4., 5., 6., + 7., nan, nan, nan, nan, nan, nan, 8., 9.}; + + QTest::newRow("fillDataHoles (change resolution)") + << std::vector{0., 1., 5., 7., 14.} << std::vector{0., 1., 2., 3., 4.} + << 1.5 << nan << nan << nan + << std::vector{0., 1., 2.5, 4., 5., 6.5, 7., 8.5, 10., 11.5, 13., 14.} + << std::vector{0., 1., nan, nan, 2., nan, 3., nan, nan, nan, nan, 4.}; + + QTest::newRow("fillDataHoles (with no data (no changes made))") + << std::vector{} << std::vector{} << 2. << nan << nan << nan + << std::vector{} << std::vector{}; + + QTest::newRow("fillDataHoles (with no resolution (no changes made))") + << std::vector{0., 1., 5., 7., 14.} << std::vector{0., 1., 2., 3., 4.} << 0. + << nan << nan << nan << std::vector{0., 1., 5., 7., 14.} + << std::vector{0., 1., 2., 3., 4.}; + + QTest::newRow("fillDataHoles (change fill value)") + << std::vector{0., 1., 5., 7., 14.} << std::vector{0., 1., 2., 3., 4.} << 2. + << -1. << nan << nan << std::vector{0., 1., 3., 5., 7., 9., 11., 13., 14.} + << std::vector{0., 1., -1., 2., 3., -1., -1., -1., 4.}; + + QTest::newRow("fillDataHoles (add data holes to the beginning)") + << std::vector{5., 7., 9., 11., 13.} << std::vector{0., 1., 2., 3., 4.} + << 2. << nan << 0. << nan << std::vector{1., 3., 5., 7., 9., 11., 13.} + << std::vector{nan, nan, 0., 1., 2., 3., 4.}; + + QTest::newRow("fillDataHoles (add data holes to the end)") + << std::vector{5., 7., 9., 11., 13.} << std::vector{0., 1., 2., 3., 4.} + << 2. << nan << nan << 21. << std::vector{5., 7., 9., 11., 13., 15., 17., 19., 21.} + << std::vector{0., 1., 2., 3., 4., nan, nan, nan, nan}; + + QTest::newRow("fillDataHoles (invalid min/max bounds (no changes made))") + << std::vector{5., 7., 9., 11., 13.} << std::vector{0., 1., 2., 3., 4.} + << 2. << nan << 8. << 13. << std::vector{5., 7., 9., 11., 13.} + << std::vector{0., 1., 2., 3., 4.}; +} + +void TestDataSeriesUtils::testFillDataHoles() +{ + QFETCH(std::vector, xAxisData); + QFETCH(std::vector, valuesData); + QFETCH(double, resolution); + QFETCH(double, fillValue); + QFETCH(double, minBound); + QFETCH(double, maxBound); + + QFETCH(std::vector, expectedXAxisData); + QFETCH(std::vector, expectedValuesData); + + // Executes method (xAxisData and valuesData are modified) + DataSeriesUtils::fillDataHoles(xAxisData, valuesData, resolution, fillValue, minBound, + maxBound); + + // Checks results + auto equal = [](const auto &data, const auto &expectedData) { + // Compares with NaN values + return std::equal(data.begin(), data.end(), expectedData.begin(), expectedData.end(), + [](const auto &val, const auto &expectedVal) { + return (std::isnan(val) && std::isnan(expectedVal)) + || val == expectedVal; + }); + }; + QVERIFY(equal(xAxisData, expectedXAxisData)); + QVERIFY(equal(valuesData, expectedValuesData)); +} + +QTEST_MAIN(TestDataSeriesUtils) +#include "TestDataSeriesUtils.moc" diff --git a/core/tests/Data/TestOptionalAxis.cpp b/core/tests/Data/TestOptionalAxis.cpp index 201495a..d4fa869 100644 --- a/core/tests/Data/TestOptionalAxis.cpp +++ b/core/tests/Data/TestOptionalAxis.cpp @@ -17,10 +17,6 @@ private slots: void testDefinedAxisCtor_data(); void testDefinedAxisCtor(); - /// Tests @sa OptionalAxis::at() method - void testAt_data(); - void testAt(); - /// Tests @sa OptionalAxis::size() method void testSize_data(); void testSize(); @@ -64,39 +60,6 @@ void TestOptionalAxis::testDefinedAxisCtor() } } -void TestOptionalAxis::testAt_data() -{ - QTest::addColumn("axis"); // Axis used for test case (defined or not) - QTest::addColumn("index"); // Index to test in the axis - QTest::addColumn("expectedValue"); // Expected axis value for the index - - OptionalAxis definedAxis{std::make_shared >(std::vector{1, 2, 3}), - Unit{"Hz"}}; - - QTest::newRow("data1") << definedAxis << 0 << 1.; - QTest::newRow("data2") << definedAxis << 1 << 2.; - QTest::newRow("data3") << definedAxis << 2 << 3.; - QTest::newRow("data4 (index out of bounds)") - << definedAxis << 3 - << std::numeric_limits::quiet_NaN(); // Expects NaN for out of bounds index - QTest::newRow("data5 (index out of bounds)") - << definedAxis << -1 - << std::numeric_limits::quiet_NaN(); // Expects NaN for out of bounds index - QTest::newRow("data6 (axis not defined)") - << OptionalAxis{} << 0 - << std::numeric_limits::quiet_NaN(); // Expects NaN for undefined axis -} - -void TestOptionalAxis::testAt() -{ - QFETCH(OptionalAxis, axis); - QFETCH(int, index); - QFETCH(double, expectedValue); - - auto value = axis.at(index); - QVERIFY((std::isnan(value) && std::isnan(expectedValue)) || value == expectedValue); -} - void TestOptionalAxis::testSize_data() { QTest::addColumn("axis"); // Axis used for test case (defined or not) diff --git a/core/tests/Data/TestScalarSeries.cpp b/core/tests/Data/TestScalarSeries.cpp index f747ff9..a79fa76 100644 --- a/core/tests/Data/TestScalarSeries.cpp +++ b/core/tests/Data/TestScalarSeries.cpp @@ -1,7 +1,7 @@ #include "Data/ScalarSeries.h" #include "DataSeriesBuilders.h" -#include "DataSeriesUtils.h" +#include "DataSeriesTestsUtils.h" #include #include @@ -9,7 +9,7 @@ /** * @brief The TestScalarSeries class defines unit tests on scalar series. * - * Most of these unit tests use generic tests defined for DataSeries (@sa DataSeriesUtils) + * Most of these unit tests use generic tests defined for DataSeries (@sa DataSeriesTestsUtils) */ class TestScalarSeries : public QObject { Q_OBJECT diff --git a/core/tests/Data/TestSpectrogramSeries.cpp b/core/tests/Data/TestSpectrogramSeries.cpp index 9c83b11..92bd3af 100644 --- a/core/tests/Data/TestSpectrogramSeries.cpp +++ b/core/tests/Data/TestSpectrogramSeries.cpp @@ -1,7 +1,7 @@ #include "Data/SpectrogramSeries.h" #include "DataSeriesBuilders.h" -#include "DataSeriesUtils.h" +#include "DataSeriesTestsUtils.h" #include #include @@ -19,7 +19,7 @@ using Components = std::vector; /** * @brief The TestSpectrogramSeries class defines unit tests on spectrogram series. * - * Most of these unit tests use generic tests defined for DataSeries (@sa DataSeriesUtils) + * Most of these unit tests use generic tests defined for DataSeries (@sa DataSeriesTestsUtils) */ class TestSpectrogramSeries : public QObject { Q_OBJECT diff --git a/core/tests/Data/TestVectorSeries.cpp b/core/tests/Data/TestVectorSeries.cpp index 3542fd9..e973e86 100644 --- a/core/tests/Data/TestVectorSeries.cpp +++ b/core/tests/Data/TestVectorSeries.cpp @@ -1,7 +1,7 @@ #include "Data/VectorSeries.h" #include "DataSeriesBuilders.h" -#include "DataSeriesUtils.h" +#include "DataSeriesTestsUtils.h" #include #include @@ -9,7 +9,7 @@ /** * @brief The TestVectorSeries class defines unit tests on vector series. * - * Most of these unit tests use generic tests defined for DataSeries (@sa DataSeriesUtils) + * Most of these unit tests use generic tests defined for DataSeries (@sa DataSeriesTestsUtils) */ class TestVectorSeries : public QObject { Q_OBJECT diff --git a/core/tests/meson.build b/core/tests/meson.build index 8020245..8df1c87 100644 --- a/core/tests/meson.build +++ b/core/tests/meson.build @@ -8,6 +8,7 @@ tests = [ [['Data/TestOneDimArrayData.cpp'],'test_1d','One Dim Array test'], [['Data/TestOptionalAxis.cpp'],'test_optional_axis','OptionalAxis test'], [['Data/TestTwoDimArrayData.cpp'],'test_2d','Two Dim Array test'], + [['Data/TestDataSeriesUtils.cpp'],'test_dataseries_util','Data series utils test'], [['DataSource/TestDataSourceController.cpp'],'test_data_source','DataSourceController test'], [['Variable/TestVariableCacheController.cpp'],'test_variable_cache','VariableCacheController test'], [['Variable/TestVariable.cpp'],'test_variable','Variable test'], @@ -17,8 +18,8 @@ tests = [ amdatest_sources = [ 'Data/DataSeriesBuilders.h', 'Data/DataSeriesBuilders.cpp', - 'Data/DataSeriesUtils.h', - 'Data/DataSeriesUtils.cpp' + 'Data/DataSeriesTestsUtils.h', + 'Data/DataSeriesTestsUtils.cpp' ] foreach unit_test : tests diff --git a/gui/src/Visualization/AxisRenderingUtils.cpp b/gui/src/Visualization/AxisRenderingUtils.cpp index 869fbbc..9c9d6ea 100644 --- a/gui/src/Visualization/AxisRenderingUtils.cpp +++ b/gui/src/Visualization/AxisRenderingUtils.cpp @@ -101,8 +101,7 @@ struct AxisSetter #include #include #include @@ -180,8 +181,7 @@ struct PlottablesUpdatersetRange(QCPRange{min, max}); @@ -204,43 +204,42 @@ struct PlottablesUpdaterx() : 0.; - auto xMax = nbX != 0 ? (its.second - 1)->x() : 0.; - - auto nbY = yAxis.size(); - auto yMin = 0., yMax = 0.; - if (nbY != 0) { - std::tie(yMin, yMax) = yAxis.bounds(); - } - colormap->data()->setSize(nbX, nbY); - colormap->data()->setRange(QCPRange{xMin, xMax}, QCPRange{yMin, yMax}); + // Computes logarithmic y-axis resolution for the spectrogram + auto yData = its.first->y(); + auto yResolution = DataSeriesUtils::resolution(yData.begin(), yData.end(), true); + + // Generates mesh for colormap + auto mesh = DataSeriesUtils::regularMesh( + its.first, its.second, DataSeriesUtils::Resolution{dataSeries.xResolution()}, + yResolution); + + dataSeries.unlock(); - // Sets values - auto xIndex = 0; - for (auto it = its.first; it != its.second; ++it, ++xIndex) { - for (auto yIndex = 0; yIndex < nbY; ++yIndex) { - auto value = it->value(yIndex); + colormap->data()->setSize(mesh.m_NbX, mesh.m_NbY); + if (!mesh.isEmpty()) { + colormap->data()->setRange( + QCPRange{mesh.m_XMin, mesh.xMax()}, + // y-axis range is converted to linear values + QCPRange{std::pow(10, mesh.m_YMin), std::pow(10, mesh.yMax())}); - colormap->data()->setCell(xIndex, yIndex, value); + // Sets values + auto index = 0; + for (auto it = mesh.m_Data.begin(), end = mesh.m_Data.end(); it != end; ++it, ++index) { + auto xIndex = index % mesh.m_NbX; + auto yIndex = index / mesh.m_NbX; - // Processing spectrogram data for display in QCustomPlot - /// For the moment, we just make the NaN values to be transparent in the colormap - /// @todo ALX: complete treatments (mesh generation, etc.) - if (std::isnan(value)) { + colormap->data()->setCell(xIndex, yIndex, *it); + + // Makes the NaN values to be transparent in the colormap + if (std::isnan(*it)) { colormap->data()->setAlpha(xIndex, yIndex, 0); } } } - dataSeries.unlock(); - // Rescales axes auto plot = colormap->parentPlot(); diff --git a/plugins/amda/include/AmdaResultParserDefs.h b/plugins/amda/include/AmdaResultParserDefs.h index 43f44dc..410201a 100644 --- a/plugins/amda/include/AmdaResultParserDefs.h +++ b/plugins/amda/include/AmdaResultParserDefs.h @@ -12,11 +12,13 @@ /// Alias to represent properties read in the header of AMDA file using Properties = QVariantHash; +extern const QString END_TIME_PROPERTY; extern const QString FILL_VALUE_PROPERTY; extern const QString MAX_BANDS_PROPERTY; extern const QString MIN_BANDS_PROPERTY; extern const QString MAX_SAMPLING_PROPERTY; extern const QString MIN_SAMPLING_PROPERTY; +extern const QString START_TIME_PROPERTY; extern const QString X_AXIS_UNIT_PROPERTY; extern const QString Y_AXIS_UNIT_PROPERTY; extern const QString VALUES_UNIT_PROPERTY; @@ -44,6 +46,9 @@ extern const QString VALUES_UNIT_PROPERTY; /// ... - Units : m/s - ... extern const QRegularExpression DEFAULT_X_AXIS_UNIT_REGEX; +/// Regex to find end time of data in a line for a spectrogram +extern const QRegularExpression SPECTROGRAM_END_TIME_REGEX; + /// Regex to find fill value used in a line for a spectrogram extern const QRegularExpression SPECTROGRAM_FILL_VALUE_REGEX; @@ -59,6 +64,9 @@ extern const QRegularExpression SPECTROGRAM_MAX_SAMPLING_REGEX; /// Regex to find min x-axis sampling in a line for a spectrogram extern const QRegularExpression SPECTROGRAM_MIN_SAMPLING_REGEX; +/// Regex to find start time of data in a line for a spectrogram +extern const QRegularExpression SPECTROGRAM_START_TIME_REGEX; + /// Regex to find y-axis unit in a line for a spectrogram extern const QRegularExpression SPECTROGRAM_Y_AXIS_UNIT_REGEX; diff --git a/plugins/amda/include/AmdaResultParserHelper.h b/plugins/amda/include/AmdaResultParserHelper.h index a362858..dbe6cbd 100644 --- a/plugins/amda/include/AmdaResultParserHelper.h +++ b/plugins/amda/include/AmdaResultParserHelper.h @@ -75,6 +75,8 @@ public: void readResultLine(const QString &line) override; private: + void handleDataHoles(); + Properties m_Properties{}; std::vector m_XAxisData{}; std::vector m_YAxisData{}; diff --git a/plugins/amda/src/AmdaResultParserDefs.cpp b/plugins/amda/src/AmdaResultParserDefs.cpp index d724a05..37c2cf9 100644 --- a/plugins/amda/src/AmdaResultParserDefs.cpp +++ b/plugins/amda/src/AmdaResultParserDefs.cpp @@ -1,10 +1,12 @@ #include "AmdaResultParserDefs.h" +const QString END_TIME_PROPERTY = QStringLiteral("endTime"); const QString FILL_VALUE_PROPERTY = QStringLiteral("fillValue"); const QString MAX_BANDS_PROPERTY = QStringLiteral("maxBands"); const QString MIN_BANDS_PROPERTY = QStringLiteral("minBands"); const QString MAX_SAMPLING_PROPERTY = QStringLiteral("maxSampling"); const QString MIN_SAMPLING_PROPERTY = QStringLiteral("minSampling"); +const QString START_TIME_PROPERTY = QStringLiteral("startTime"); const QString X_AXIS_UNIT_PROPERTY = QStringLiteral("xAxisUnit"); const QString Y_AXIS_UNIT_PROPERTY = QStringLiteral("yAxisUnit"); const QString VALUES_UNIT_PROPERTY = QStringLiteral("valuesUnit"); @@ -12,6 +14,9 @@ const QString VALUES_UNIT_PROPERTY = QStringLiteral("valuesUnit"); const QRegularExpression DEFAULT_X_AXIS_UNIT_REGEX = QRegularExpression{QStringLiteral("-\\s*Units\\s*:\\s*(.+?)\\s*-")}; +const QRegularExpression SPECTROGRAM_END_TIME_REGEX + = QRegularExpression{QStringLiteral("\\s*INTERVAL_STOP\\s*:\\s*(.*)")}; + const QRegularExpression SPECTROGRAM_FILL_VALUE_REGEX = QRegularExpression{QStringLiteral("\\s*PARAMETER_FILL_VALUE\\s*:\\s*(.*)")}; @@ -27,6 +32,9 @@ const QRegularExpression SPECTROGRAM_MAX_SAMPLING_REGEX const QRegularExpression SPECTROGRAM_MIN_SAMPLING_REGEX = QRegularExpression{QStringLiteral("\\s*DATASET_MIN_SAMPLING\\s*:\\s*(.*)")}; +const QRegularExpression SPECTROGRAM_START_TIME_REGEX + = QRegularExpression{QStringLiteral("\\s*INTERVAL_START\\s*:\\s*(.*)")}; + const QRegularExpression SPECTROGRAM_Y_AXIS_UNIT_REGEX = QRegularExpression{QStringLiteral("\\s*PARAMETER_TABLE_UNITS\\[0\\]\\s*:\\s*(.*)")}; diff --git a/plugins/amda/src/AmdaResultParserHelper.cpp b/plugins/amda/src/AmdaResultParserHelper.cpp index b463a43..a483800 100644 --- a/plugins/amda/src/AmdaResultParserHelper.cpp +++ b/plugins/amda/src/AmdaResultParserHelper.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -158,6 +159,18 @@ bool tryReadProperty(Properties &properties, const QString &key, const QString & } /** + * Reads a line from the AMDA file and tries to extract a data from it. Date is converted to double + * @sa tryReadProperty() + */ +bool tryReadDate(Properties &properties, const QString &key, const QString &line, + const QRegularExpression ®ex, bool timeUnit = false) +{ + return tryReadProperty(properties, key, line, regex, [timeUnit](const auto &match) { + return QVariant::fromValue(doubleDate(match.captured(1))); + }); +} + +/** * Reads a line from the AMDA file and tries to extract a double from it * @sa tryReadProperty() */ @@ -263,7 +276,7 @@ bool SpectrogramParserHelper::checkProperties() auto minBands = m_Properties.value(MIN_BANDS_PROPERTY).value >(); auto maxBands = m_Properties.value(MAX_BANDS_PROPERTY).value >(); - if (minBands.size() != maxBands.size()) { + if (minBands.size() < 2 || minBands.size() != maxBands.size()) { qCWarning(LOG_AmdaResultParserHelper()) << QObject::tr( "Can't generate y-axis data from bands extracted: bands intervals are invalid"); return false; @@ -283,18 +296,20 @@ bool SpectrogramParserHelper::checkProperties() // Sets fill value m_FillValue = m_Properties.value(FILL_VALUE_PROPERTY).value(); - /// @todo: handle min/max samplings? - return true; } std::shared_ptr SpectrogramParserHelper::createSeries() { + // Before creating the series, we handle its data holes + handleDataHoles(); + return std::make_shared( std::move(m_XAxisData), std::move(m_YAxisData), std::move(m_ValuesData), Unit{"t", true}, // x-axis unit is always a time unit m_Properties.value(Y_AXIS_UNIT_PROPERTY).value(), - m_Properties.value(VALUES_UNIT_PROPERTY).value()); + m_Properties.value(VALUES_UNIT_PROPERTY).value(), + m_Properties.value(MIN_SAMPLING_PROPERTY).value()); } void SpectrogramParserHelper::readPropertyLine(const QString &line) @@ -337,6 +352,15 @@ void SpectrogramParserHelper::readPropertyLine(const QString &line) [&] { return tryReadDoubles(m_Properties, MAX_BANDS_PROPERTY, line, SPECTROGRAM_MAX_BANDS_REGEX); + }, + // start time of data + [&] { + return tryReadDate(m_Properties, START_TIME_PROPERTY, line, + SPECTROGRAM_START_TIME_REGEX); + }, + // end time of data + [&] { + return tryReadDate(m_Properties, END_TIME_PROPERTY, line, SPECTROGRAM_END_TIME_REGEX); }}; for (auto function : functions) { @@ -352,6 +376,18 @@ void SpectrogramParserHelper::readResultLine(const QString &line) tryReadResult(m_XAxisData, m_ValuesData, line, m_ValuesIndexes, m_FillValue); } +void SpectrogramParserHelper::handleDataHoles() +{ + // Fills data holes according to the max resolution found in the AMDA file + auto resolution = m_Properties.value(MAX_SAMPLING_PROPERTY).value(); + auto fillValue = m_Properties.value(FILL_VALUE_PROPERTY).value(); + auto minBound = m_Properties.value(START_TIME_PROPERTY).value(); + auto maxBound = m_Properties.value(END_TIME_PROPERTY).value(); + + DataSeriesUtils::fillDataHoles(m_XAxisData, m_ValuesData, resolution, fillValue, minBound, + maxBound); +} + // ////////////////// // // VectorParserHelper // // ////////////////// // diff --git a/plugins/amda/tests-resources/TestAmdaResultParser/spectro/ValidSpectrogramDataHoles.txt b/plugins/amda/tests-resources/TestAmdaResultParser/spectro/ValidSpectrogramDataHoles.txt new file mode 100644 index 0000000..81b526b --- /dev/null +++ b/plugins/amda/tests-resources/TestAmdaResultParser/spectro/ValidSpectrogramDataHoles.txt @@ -0,0 +1,62 @@ +# ----------- +# AMDA INFO : +# ----------- +# AMDA_ABOUT : Created by CDPP/AMDA(c) +# AMDA_VERSION : 3.5.0 +# AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team +# +# -------------- +# REQUEST INFO : +# -------------- +# REQUEST_STRUCTURE : one-file-per-parameter-per-interval +# REQUEST_TIME_FORMAT : ISO 8601 +# REQUEST_OUTPUT_PARAMS : tha_ion_sp +# +# ----------------- +# BASE PARAMETERS : +# ----------------- +# +# MISSION_ID : NONE +# +# INSTRUMENT_ID : NONE +# +# DATASET_ID : tha-esa-l2i +# DATASET_NAME : ion full mode +# DATASET_SOURCE : CDPP/DDServer +# DATASET_GLOBAL_START : 2007-03-07T18:53:59.134 +# DATASET_GLOBAL_STOP : 2017-11-04T18:35:25.907 +# DATASET_MIN_SAMPLING : 96 +# DATASET_MAX_SAMPLING : 180 +# +# PARAMETER_ID : tha_ion_sp +# PARAMETER_NAME : tha_ion_sp +# PARAMETER_SHORT_NAME : spectra +# PARAMETER_UNITS : eV/(cm^2-s-sr-eV) +# PARAMETER_TENSOR_ORDER : 0 +# PARAMETER_TABLE[0] : energy +# PARAMETER_TABLE_UNITS[0] : eV +# PARAMETER_TABLE_MIN_VALUES[0] : 14234.4,18737.3 +# PARAMETER_TABLE_MAX_VALUES[0] : 18737.3,23254.9 +# PARAMETER_FILL_VALUE : nan +# PARAMETER_UCD : phys.flux.density;phys.energy;phys.atmol.ionStage +# +# +# --------------- +# INTERVAL INFO : +# --------------- +# INTERVAL_START : 2011-12-10T12:10:00.000 +# INTERVAL_STOP : 2011-12-10T12:40:00.000 +# +# ------ +# DATA : +# ------ +# DATA_COLUMNS : AMDA_TIME, tha_ion_sp[0], tha_ion_sp[1] +# +2011-12-10T12:10:54.000 2577578.000 2336016.000 +2011-12-10T12:17:23.000 2314121.500 1712093.125 +2011-12-10T12:23:51.000 2063608.750 1614491.625 +2011-12-10T12:30:19.000 2234525.500 1764516.500 +2011-12-10T12:35:04.000 1670215.250 1688078.500 +2011-12-10T12:36:41.000 1689243.250 1743183.500 +2011-12-10T12:38:18.000 1654617.125 1733603.250 +2011-12-10T12:39:55.000 1504983.750 1708356.500 \ No newline at end of file diff --git a/plugins/amda/tests-resources/TestAmdaResultParser/spectro/ValidSpectrogramDataHoles2.txt b/plugins/amda/tests-resources/TestAmdaResultParser/spectro/ValidSpectrogramDataHoles2.txt new file mode 100644 index 0000000..e5db048 --- /dev/null +++ b/plugins/amda/tests-resources/TestAmdaResultParser/spectro/ValidSpectrogramDataHoles2.txt @@ -0,0 +1,62 @@ +# ----------- +# AMDA INFO : +# ----------- +# AMDA_ABOUT : Created by CDPP/AMDA(c) +# AMDA_VERSION : 3.5.0 +# AMDA_ACKNOWLEDGEMENT : CDPP/AMDA Team +# +# -------------- +# REQUEST INFO : +# -------------- +# REQUEST_STRUCTURE : one-file-per-parameter-per-interval +# REQUEST_TIME_FORMAT : ISO 8601 +# REQUEST_OUTPUT_PARAMS : tha_ion_sp +# +# ----------------- +# BASE PARAMETERS : +# ----------------- +# +# MISSION_ID : NONE +# +# INSTRUMENT_ID : NONE +# +# DATASET_ID : tha-esa-l2i +# DATASET_NAME : ion full mode +# DATASET_SOURCE : CDPP/DDServer +# DATASET_GLOBAL_START : 2007-03-07T18:53:59.134 +# DATASET_GLOBAL_STOP : 2017-11-04T18:35:25.907 +# DATASET_MIN_SAMPLING : 96 +# DATASET_MAX_SAMPLING : 240 +# +# PARAMETER_ID : tha_ion_sp +# PARAMETER_NAME : tha_ion_sp +# PARAMETER_SHORT_NAME : spectra +# PARAMETER_UNITS : eV/(cm^2-s-sr-eV) +# PARAMETER_TENSOR_ORDER : 0 +# PARAMETER_TABLE[0] : energy +# PARAMETER_TABLE_UNITS[0] : eV +# PARAMETER_TABLE_MIN_VALUES[0] : 14234.4,18737.3 +# PARAMETER_TABLE_MAX_VALUES[0] : 18737.3,23254.9 +# PARAMETER_FILL_VALUE : nan +# PARAMETER_UCD : phys.flux.density;phys.energy;phys.atmol.ionStage +# +# +# --------------- +# INTERVAL INFO : +# --------------- +# INTERVAL_START : 2011-12-10T12:00:00.000 +# INTERVAL_STOP : 2011-12-10T13:00:00.000 +# +# ------ +# DATA : +# ------ +# DATA_COLUMNS : AMDA_TIME, tha_ion_sp[0], tha_ion_sp[1] +# +2011-12-10T12:10:54.000 2577578.000 2336016.000 +2011-12-10T12:17:23.000 2314121.500 1712093.125 +2011-12-10T12:23:51.000 2063608.750 1614491.625 +2011-12-10T12:30:19.000 2234525.500 1764516.500 +2011-12-10T12:35:04.000 1670215.250 1688078.500 +2011-12-10T12:36:41.000 1689243.250 1743183.500 +2011-12-10T12:38:18.000 1654617.125 1733603.250 +2011-12-10T12:39:55.000 1504983.750 1708356.500 \ No newline at end of file diff --git a/plugins/amda/tests/TestAmdaResultParser.cpp b/plugins/amda/tests/TestAmdaResultParser.cpp index 8aa3ad8..ca76006 100644 --- a/plugins/amda/tests/TestAmdaResultParser.cpp +++ b/plugins/amda/tests/TestAmdaResultParser.cpp @@ -135,11 +135,10 @@ struct ExpectedResults { QCOMPARE(yAxis.unit(), m_YAxisUnit); // Data - auto yAxisSize = yAxis.size(); - QCOMPARE(yAxisSize, m_YAxisData.size()); - for (auto i = 0; i < yAxisSize; ++i) { - QCOMPARE(yAxis.at(i), m_YAxisData.at(i)); - } + QVERIFY(std::equal(yAxis.cbegin(), yAxis.cend(), m_YAxisData.cbegin(), + m_YAxisData.cend(), [](const auto &it, const auto &expectedVal) { + return it.first() == expectedVal; + })); } } else { @@ -397,6 +396,135 @@ void TestAmdaResultParser::testReadSpectrogramTxt_data() << QStringLiteral("spectro/ValidSpectrogramFillValues.txt") << nanValuesResult; // Fill values are replaced by NaN values in the data series + QTest::newRow("Valid file (containing data holes, resolution = 3 minutes)") + << QStringLiteral("spectro/ValidSpectrogramDataHoles.txt") + << ExpectedResults{} + .setParsingOK(true) + .setXAxisUnit(Unit{"t", true}) + .setXAxisData({dateTime(2011, 12, 10, 12, 10, 54), // + dateTime(2011, 12, 10, 12, 13, 54), // Data hole + dateTime(2011, 12, 10, 12, 16, 54), // Data hole + dateTime(2011, 12, 10, 12, 17, 23), // + dateTime(2011, 12, 10, 12, 20, 23), // Data hole + dateTime(2011, 12, 10, 12, 23, 23), // Data hole + dateTime(2011, 12, 10, 12, 23, 51), // + dateTime(2011, 12, 10, 12, 26, 51), // Data hole + dateTime(2011, 12, 10, 12, 29, 51), // Data hole + dateTime(2011, 12, 10, 12, 30, 19), // + dateTime(2011, 12, 10, 12, 33, 19), // Data hole + dateTime(2011, 12, 10, 12, 35, 04), // + dateTime(2011, 12, 10, 12, 36, 41), // + dateTime(2011, 12, 10, 12, 38, 18), // + dateTime(2011, 12, 10, 12, 39, 55)}) + .setYAxisEnabled(true) + .setYAxisUnit(Unit{"eV"}) + .setYAxisData({16485.85, 20996.1}) // middle of the intervals of each band + .setValuesUnit(Unit{"eV/(cm^2-s-sr-eV)"}) + .setValuesData(QVector >{{2577578.000, // + nan, // Data hole + nan, // Data hole + 2314121.500, // + nan, // Data hole + nan, // Data hole + 2063608.750, // + nan, // Data hole + nan, // Data hole + 2234525.500, // + nan, // Data hole + 1670215.250, // + 1689243.250, // + 1654617.125, // + 1504983.750}, + {2336016.000, // + nan, // Data hole + nan, // Data hole + 1712093.125, // + nan, // Data hole + nan, // Data hole + 1614491.625, // + nan, // Data hole + nan, // Data hole + 1764516.500, // + nan, // Data hole + 1688078.500, // + 1743183.500, // + 1733603.250, // + 1708356.500}}); + + QTest::newRow( + "Valid file (containing data holes at the beginning and the end, resolution = 4 minutes)") + << QStringLiteral("spectro/ValidSpectrogramDataHoles2.txt") + << ExpectedResults{} + .setParsingOK(true) + .setXAxisUnit(Unit{"t", true}) + .setXAxisData({ + dateTime(2011, 12, 10, 12, 2, 54), // Data hole + dateTime(2011, 12, 10, 12, 6, 54), // Data hole + dateTime(2011, 12, 10, 12, 10, 54), // + dateTime(2011, 12, 10, 12, 14, 54), // Data hole + dateTime(2011, 12, 10, 12, 17, 23), // + dateTime(2011, 12, 10, 12, 21, 23), // Data hole + dateTime(2011, 12, 10, 12, 23, 51), // + dateTime(2011, 12, 10, 12, 27, 51), // Data hole + dateTime(2011, 12, 10, 12, 30, 19), // + dateTime(2011, 12, 10, 12, 34, 19), // Data hole + dateTime(2011, 12, 10, 12, 35, 04), // + dateTime(2011, 12, 10, 12, 36, 41), // + dateTime(2011, 12, 10, 12, 38, 18), // + dateTime(2011, 12, 10, 12, 39, 55), + dateTime(2011, 12, 10, 12, 43, 55), // Data hole + dateTime(2011, 12, 10, 12, 47, 55), // Data hole + dateTime(2011, 12, 10, 12, 51, 55), // Data hole + dateTime(2011, 12, 10, 12, 55, 55), // Data hole + dateTime(2011, 12, 10, 12, 59, 55) // Data hole + }) + .setYAxisEnabled(true) + .setYAxisUnit(Unit{"eV"}) + .setYAxisData({16485.85, 20996.1}) // middle of the intervals of each band + .setValuesUnit(Unit{"eV/(cm^2-s-sr-eV)"}) + .setValuesData(QVector >{{ + nan, // Data hole + nan, // Data hole + 2577578.000, // + nan, // Data hole + 2314121.500, // + nan, // Data hole + 2063608.750, // + nan, // Data hole + 2234525.500, // + nan, // Data hole + 1670215.250, // + 1689243.250, // + 1654617.125, // + 1504983.750, // + nan, // Data hole + nan, // Data hole + nan, // Data hole + nan, // Data hole + nan // Data hole + }, + { + nan, // Data hole + nan, // Data hole + 2336016.000, // + nan, // Data hole + 1712093.125, // + nan, // Data hole + 1614491.625, // + nan, // Data hole + 1764516.500, // + nan, // Data hole + 1688078.500, // + 1743183.500, // + 1733603.250, // + 1708356.500, // + nan, // Data hole + nan, // Data hole + nan, // Data hole + nan, // Data hole + nan // Data hole + }}); + // Invalid files QTest::newRow("Invalid file (inconsistent bands)") << QStringLiteral("spectro/InvalidSpectrogramWrongBands.txt")