HG_REPOSITORIES/LPP/INSTRUMENTATION/USERS/JEANDET/SciQLOPCore Commit - r17:895ab1d87afd

Basic asynchronous variable update, still a lot to do...

jeandet -

r17:895ab1d87afd

parent child

Context file:

r17:895ab1d87afd

include/Variable/private/VCTransaction.h created 644 +53 0

@@ -0,0 +1,53
	1	#include <QThreadPool>
	2	#include <QRunnable>
	3	#include <QObject>
	4	#include <QReadWriteLock>
	5
	6	#include "Variable/VariableSynchronizationGroup2.h"
	7	#include <Variable/Variable.h>
	8	#include <Common/containers.h>
	9	#include <Common/debug.h>
	10	#include <Data/DataProviderParameters.h>
	11	#include <Data/DateTimeRangeHelper.h>
	12	#include <Data/DateTimeRange.h>
	13	#include <Data/IDataProvider.h>
	14
	15	struct VCTransaction
	16	{
	17	VCTransaction(QUuid refVar, DateTimeRange range, int ready)
	18	:refVar{refVar},range{range},ready{ready}
	19	{}
	20	QUuid refVar;
	21	DateTimeRange range;
	22	int ready;
	23	QReadWriteLock lock;
	24	};
	25
	26	class TransactionExe:public QObject,public QRunnable
	27	{
	28	Q_OBJECT
	29	std::shared_ptr<Variable> _variable;
	30	std::shared_ptr<IDataProvider> _provider;
	31	std::vector<DateTimeRange> _ranges;
	32	DateTimeRange _range;
	33	DateTimeRange _cacheRange;
	34	public:
	35	TransactionExe(const std::shared_ptr<Variable>& variable, const std::shared_ptr<IDataProvider>& provider,
	36	const std::vector<DateTimeRange>& ranges, DateTimeRange range, DateTimeRange cacheRange)
	37	:_variable{variable}, _provider{provider},_ranges{ranges},_range{range},_cacheRange{cacheRange}
	38	{
	39	setAutoDelete(true);
	40	}
	41	void run()override
	42	{
	43	std::vector<IDataSeries*> data;
	44	for(auto range:_ranges)
	45	{
	46	data.push_back(_provider->getData(DataProviderParameters{{range}, _variable->metadata()}));
	47	}
	48	_variable->updateData(data, _range, _cacheRange, true);
	49	emit transactionComplete();
	50	}
	51	signals:
	52	void transactionComplete();
	53	};

tests/Variable/TestVariableController2Async.cpp created 644 +67 0

@@ -0,0 +1,67
	1	#include <cmath>
	2	#include <algorithm>
	3	#include <numeric>
	4	#include <QtTest>
	5	#include <QObject>
	6	#include <Variable/VariableController2.h>
	7	#include <Data/DateTimeRange.h>
	8	#include <Data/IDataProvider.h>
	9	#include <Data/ScalarSeries.h>
	10	#include <Data/DataProviderParameters.h>
	11	#include <Common/containers.h>
	12
	13	#include <TestUtils/TestProviders.h>
	14
	15	#define TEST_VC2_FIXTURE(slope) \
	16	VariableController2 vc; \
	17	auto provider = std::make_shared<SimpleRange<slope>>();\
	18
	19	#define TEST_VC2_CREATE_DEFAULT_VARS(name1, name2, name3)\
	20	auto range = DateTimeRange::fromDateTime(QDate(2018,8,7),QTime(14,00),\
	21	QDate(2018,8,7),QTime(16,00));\
	22	auto name1 = vc.createVariable("name1", {}, provider, range);\
	23	auto name2 = vc.createVariable("name1", {}, provider, range);\
	24	auto name3 = vc.createVariable("name1", {}, provider, range);\
	25	vc.synchronize(name1,name2);\
	26
	27
	28	class TestVariableController2Async : public QObject
	29	{
	30	Q_OBJECT
	31	public:
	32	explicit TestVariableController2Async(QObject *parent = nullptr) : QObject(parent){}
	33	signals:
	34
	35	private slots:
	36	void initTestCase(){}
	37	void cleanupTestCase(){}
	38
	39	void testSimplePan()
	40	{
	41	TEST_VC2_FIXTURE(2);
	42	auto range = DateTimeRange::fromDateTime(QDate(2018,8,7),QTime(14,00),
	43	QDate(2018,8,7),QTime(16,00));
	44	int variableUpdated=0;
	45	auto var1 = vc.createVariable("var1", {}, provider, range);
	46	auto var2 = vc.createVariable("var2", {}, provider, range);
	47	auto var3 = vc.createVariable("var3", {}, provider, range);
	48	connect(&(*var2),&Variable::updated, [&variableUpdated](){variableUpdated+=1;});
	49	vc.synchronize(var1,var2);
	50	vc.asyncChangeRange(var1,range+Seconds<double>{10000.});
	51	vc.asyncChangeRange(var1,range+Seconds<double>{50000.});
	52	vc.asyncChangeRange(var1,range+Seconds<double>{100000.});
	53	vc.asyncChangeRange(var1,range+Seconds<double>{150000.});
	54	while(!vc.isReady(var1) \|\| !vc.isReady(var2))
	55	{
	56	QCoreApplication::processEvents();
	57	}
	58	}
	59
	60
	61	};
	62
	63
	64	QTEST_MAIN(TestVariableController2Async)
	65
	66	#include "TestVariableController2Async.moc"
	67

CMakeLists.txt +1 0

             cmake_minimum_required(VERSION 3.6)
             project(SciQLOPCore CXX)
             OPTION (CPPCHECK "Analyzes the source code with cppcheck" OFF)
             OPTION (CLANG_TIDY "Analyzes the source code with Clang Tidy" OFF)
             OPTION (IWYU "Analyzes the source code with Include What You Use" OFF)
             OPTION (Catalog "builds catalog API" OFF)
             set(CMAKE_CXX_STANDARD 17)
             set(CMAKE_AUTOMOC ON)
             #https://gitlab.kitware.com/cmake/cmake/issues/15227
             #set(CMAKE_AUTOUIC ON)
             if(POLICY CMP0071)
                 cmake_policy(SET CMP0071 OLD)
             endif()
             set(CMAKE_AUTORCC ON)
             set(CMAKE_INCLUDE_CURRENT_DIR ON)
             find_package(Qt5 COMPONENTS Core Widgets Network PrintSupport Svg Test REQUIRED)
             find_package(pybind11 CONFIG QUIET)
             if (NOT pybind11_FOUND)
                 execute_process(COMMAND git submodule init external/pybind11 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
                 execute_process(COMMAND git submodule update external/pybind11 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
                 add_subdirectory(external/pybind11)
             endif()
             macro(declare_test testname testexe sources libraries)
                 add_executable(${testexe} ${sources})
                 target_link_libraries(${testexe} ${libraries})
                 add_test(NAME ${testname} COMMAND ${testexe})
             endmacro(declare_test)
             enable_testing()
             FILE (GLOB_RECURSE core_SRCS
                 ./include/DataSource/DataSourceItemMergeHelper.h
                 ./include/DataSource/DataSourceItemAction.h
                 ./include/DataSource/DataSourceItem.h
                 ./include/DataSource/DataSourceController.h
                 ./include/Common/SortUtils.h
                 ./include/Common/spimpl.h
                 ./include/Common/MimeTypesDef.h
                 ./include/Common/MetaTypes.h
                 ./include/Common/StringUtils.h
                 ./include/Common/SignalWaiter.h
                 ./include/Common/DateUtils.h
                 ./include/Common/Numeric.h
                 ./include/Common/deprecate.h
                 ./include/Common/containers.h
                 ./include/Common/debug.h
                 ./include/Plugin/IPlugin.h
                 ./include/Data/ArrayDataIterator.h
                 ./include/Data/VariableRequest.h
                 ./include/Data/VectorSeries.h
                 ./include/Data/DateTimeRange.h
                 ./include/Data/DateTimeRangeHelper.h
                 ./include/Data/ScalarSeries.h
                 ./include/Data/DataSeriesMergeHelper.h
                 ./include/Data/DataSeries.h
                 ./include/Data/AcquisitionDataPacket.h
                 ./include/Data/DataSeriesType.h
                 ./include/Data/AcquisitionRequest.h
                 ./include/Data/SqpIterator.h
                 ./include/Data/ArrayData.h
                 ./include/Data/DataSeriesIterator.h
                 ./include/Data/DataSeriesUtils.h
                 ./include/Data/SpectrogramSeries.h
                 ./include/Data/Unit.h
                 ./include/Data/DataProviderParameters.h
                 ./include/Data/OptionalAxis.h
                 ./include/Data/IDataProvider.h
                 ./include/Data/IDataSeries.h
                 ./include/Network/NetworkController.h
                 ./include/Network/Downloader.h
                 ./include/Version.h
                 ./include/CoreGlobal.h
                 ./include/Visualization/VisualizationController.h
                 ./include/PluginManager/PluginManager.h
                 ./include/Variable/VariableModel.h
                 ./include/Variable/VariableAcquisitionWorker.h
                 ./include/Variable/VariableCacheStrategy.h
                 ./include/Variable/VariableSynchronizationGroup.h
                 ./include/Variable/VariableSynchronizationGroup2.h
                 ./include/Variable/ProportionalCacheStrategy.h
                 ./include/Variable/SingleThresholdCacheStrategy.h
                 ./include/Variable/VariableCacheStrategyFactory.h
                 ./include/Variable/Variable.h
                 ./include/Variable/VariableCacheController.h
                 ./include/Variable/VariableController.h
                 ./include/Variable/VariableController2.h
+                ./include/Variable/private/VCTransaction.h
                 ./include/Time/TimeController.h
                 ./include/Settings/ISqpSettingsBindable.h
                 ./include/Settings/SqpSettingsDefs.h
                 ./src/DataSource/DataSourceItem.cpp
                 ./src/DataSource/DataSourceItemAction.cpp
                 ./src/DataSource/DataSourceItemMergeHelper.cpp
                 ./src/DataSource/DataSourceController.cpp
                 ./src/Common/DateUtils.cpp
                 ./src/Common/MimeTypesDef.cpp
                 ./src/Common/StringUtils.cpp
                 ./src/Common/SignalWaiter.cpp
                 ./src/Data/ScalarSeries.cpp
                 ./src/Data/DataSeriesIterator.cpp
                 ./src/Data/OptionalAxis.cpp
                 ./src/Data/ArrayDataIterator.cpp
                 ./src/Data/SpectrogramSeries.cpp
                 ./src/Data/DataSeriesUtils.cpp
                 ./src/Data/VectorSeries.cpp
                 ./src/Network/NetworkController.cpp
                 ./src/Network/Downloader.cpp
                 ./src/Visualization/VisualizationController.cpp
                 ./src/PluginManager/PluginManager.cpp
                 ./src/Variable/VariableController.cpp
                 ./src/Variable/VariableController2.cpp
                 ./src/Variable/VariableModel.cpp
                 ./src/Variable/VariableCacheController.cpp
                 ./src/Variable/VariableSynchronizationGroup.cpp
                 ./src/Variable/VariableSynchronizationGroup2.cpp
                 ./src/Variable/Variable.cpp
                 ./src/Variable/VariableAcquisitionWorker.cpp
                 ./src/Version.cpp
                 ./src/Time/TimeController.cpp
                 ./src/Settings/SqpSettingsDefs.cpp
                     )
             IF(Catalog)
                 FILE (GLOB_RECURSE core_catalog_SRCS
                     ./src/Catalogue/CatalogueController.cpp
                     ./include/Catalogue/CatalogueController.h
                     )
             ELSE()
             FILE (GLOB_RECURSE core_catalog_SRCS
                 )
             ENDIF(Catalog)
             add_definitions(-DCORE_STATIC)
             #add_definitions(-DHIDE_DEPRECATED)
             add_definitions(-DSCIQLOP_CRASH_ON_ERROR)
             add_library(sciqlopcore ${core_SRCS} ${core_catalog_SRCS})
             SET_TARGET_PROPERTIES(sciqlopcore PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
             target_include_directories(sciqlopcore PUBLIC
                 $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
                 $<INSTALL_INTERFACE:include/SciQLOP>
             )
             target_link_libraries(sciqlopcore PUBLIC
               Qt5::Core
               Qt5::Network
             )
             if(Catalog)
                 target_link_libraries(sciqlopcore PUBLIC
                   catalogs
                 )
             endif()
             pybind11_add_module(sciqlopqt src/pybind11_wrappers/QtWrappers.cpp)
             target_link_libraries(sciqlopqt PUBLIC Qt5::Core)
             pybind11_add_module(pysciqlopcore src/pybind11_wrappers/CoreWrappers.cpp)
             target_link_libraries(pysciqlopcore PUBLIC sciqlopcore)
             add_library(pysciqlop src/pybind11_wrappers/pywrappers_common.h)
             target_link_libraries(pysciqlop PUBLIC Qt5::Core)
             target_include_directories(pysciqlop PUBLIC
                 $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/pybind11_wrappers/>
                 $<INSTALL_INTERFACE:include/SciQLOP/py_wrappers>
             )
             SET_PROPERTY(GLOBAL PROPERTY CORE_PYTHON_PATH  ${CMAKE_CURRENT_BINARY_DIR})
             install(TARGETS sciqlopcore EXPORT SciQLOPCoreConfig
                 ARCHIVE  DESTINATION ${CMAKE_INSTALL_LIBDIR}
                 LIBRARY  DESTINATION ${CMAKE_INSTALL_LIBDIR}
                 RUNTIME  DESTINATION ${CMAKE_INSTALL_BINDIR})
             install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/SciQLOP)
             install(EXPORT SciQLOPCoreConfig DESTINATION share/SciQLOPCore/cmake)
             export(TARGETS sciqlopcore FILE SciQLOPCoreConfig.cmake)
             add_subdirectory(tests)

include/Data/DateTimeRange.h +5 0

             #ifndef SCIQLOP_DATETIMERANGE_H
             #define SCIQLOP_DATETIMERANGE_H
             #include <cmath>
             #include <QObject>
             #include <QDebug>
             #include <opaque/numeric_typedef.hpp>
             #include <Common/DateUtils.h>
             #include <Common/MetaTypes.h>
             #include <Common/Numeric.h>
             template <typename T>
             struct Seconds : opaque::numeric_typedef<T, Seconds<T>> ,
                     opaque::binop::multipliable     <Seconds<T>, true , Seconds<T>, T, T>,
                     opaque::binop::dividable        <Seconds<T>, true , Seconds<T>, T, T>,
                     opaque::binop::addable          <Seconds<T>, true , Seconds<T>, T, T>,
                     opaque::binop::subtractable     <Seconds<T>, true , Seconds<T>, T, T>
             {
               using base  = opaque::numeric_typedef<T, Seconds<T>>;
               using base::base;
               operator T () const {return this->value;}
             };
             struct InvalidDateTimeRangeTransformation{};
             struct DateTimeRangeTransformation
             {
                 double zoom;
                 Seconds<double> shift;
                 bool operator==(const DateTimeRangeTransformation& other) const
                 {
                     return SciQLop::numeric::almost_equal(zoom, other.zoom, 1) &&
                             SciQLop::numeric::almost_equal<double>(shift, other.shift, 1);
                 }
+                DateTimeRangeTransformation merge(const DateTimeRangeTransformation& other) const
+                {
+                    return DateTimeRangeTransformation{zoom*other.zoom,shift+other.shift};
+                }
             };
             /**
              * @brief The SqpRange struct holds the information of time parameters
              */
             struct DateTimeRange {
                 DateTimeRange()
                     :m_TStart(std::nan("")), m_TEnd(std::nan(""))
                 {}
                 DateTimeRange(double TStart, double TEnd)
                     :m_TStart(TStart), m_TEnd(TEnd)
                 {}
                 /// Creates SqpRange from dates and times
                 static DateTimeRange fromDateTime(const QDate &startDate, const QTime &startTime,
                                              const QDate &endDate, const QTime &endTime)
                 {
                     return {DateUtils::secondsSinceEpoch(QDateTime{startDate, startTime, Qt::UTC}),
                             DateUtils::secondsSinceEpoch(QDateTime{endDate, endTime, Qt::UTC})};
                 }
                 static DateTimeRange fromDateTime(const QDateTime &start,  const QDateTime &end)
                 {
                     return {DateUtils::secondsSinceEpoch(start),
                             DateUtils::secondsSinceEpoch(end)};
                 }
                 /// Start time (UTC)
                 double m_TStart;
                 /// End time (UTC)
                 double m_TEnd;
                 Seconds<double> delta()const noexcept{return Seconds<double>{this->m_TEnd - this->m_TStart};}
                 bool contains(const DateTimeRange &dateTime) const noexcept
                 {
                     return (m_TStart <= dateTime.m_TStart && m_TEnd >= dateTime.m_TEnd);
                 }
                 Seconds<double> center() const noexcept
                 {
                     return Seconds<double>((m_TStart + m_TEnd) / 2.);
                 }
                 bool intersect(const DateTimeRange &dateTime) const noexcept
                 {
                     return (m_TEnd >= dateTime.m_TStart && m_TStart <= dateTime.m_TEnd);
                 }
                 inline DateTimeRange transform(const DateTimeRangeTransformation& tr)const noexcept;
                 bool operator==(const DateTimeRange &other) const
                 {
                     return SciQLop::numeric::almost_equal(m_TStart, other.m_TStart, 1) &&
                             SciQLop::numeric::almost_equal(m_TEnd, other.m_TEnd, 1);
                 }
                 bool operator!=(const DateTimeRange &other) const { return !(*this == other); }
                 void grow(double factor)noexcept
                 {
                     double grow_v{delta()*(factor - 1.)/2.};
                     m_TStart -= grow_v;
                     m_TEnd += grow_v;
                 }
                 void shrink(double factor)noexcept
                 {
                     double shrink_v{this->delta()*(1. - factor)/2.};
                     m_TStart += shrink_v;
                     m_TEnd -= shrink_v;
                 }
                 DateTimeRange& operator*=(double k)
                 {
                     this->grow(k);
                     return *this;
                 }
                 DateTimeRange&  operator/=(double k)
                 {
                     this->shrink(k);
                     return *this;
                 }
                 // compute set difference
                 std::vector<DateTimeRange> operator-(const DateTimeRange& other)const
                 {
                     std::vector<DateTimeRange> result;
                     if(std::isnan(other.m_TStart)||std::isnan(other.m_TEnd)||!this->intersect(other))
                     {
                         result.emplace_back(m_TStart, m_TEnd);
                     }
                     else
                     {
                         if(this->m_TStart<other.m_TStart)
                         {
                             result.emplace_back(this->m_TStart, other.m_TStart);
                         }
                         if(this->m_TEnd>other.m_TEnd)
                         {
                             result.emplace_back(this->m_TEnd, other.m_TEnd);
                         }
                     }
                     return result;
                 }
             };
             template <class T>
             DateTimeRange& operator+=(DateTimeRange&r, Seconds<T> offset)
             {
                 shift(r,offset);
                 return r;
             }
             template <class T>
             DateTimeRange&  operator-=(DateTimeRange&r, Seconds<T> offset)
             {
                 shift(r,-offset);
+                return r;
             }
             template <class T>
             void shift(DateTimeRange& r, Seconds<T> offset)
             {
                 r.m_TEnd+=static_cast<double>(offset);
                 r.m_TStart+=static_cast<double>(offset);
             }
             inline DateTimeRange operator*(const DateTimeRange& r, double k)
             {
                 DateTimeRange result{r};
                 result.grow(k);
                 return result;
             }
             inline DateTimeRange operator/(const DateTimeRange& r, double k)
             {
                 DateTimeRange result{r};
                 result.shrink(k);
                 return result;
             }
             template<class T>
             DateTimeRange operator+(const DateTimeRange& r, Seconds<T> offset)
             {
                 DateTimeRange result{r};
                 shift(result,offset);
                 return result;
             }
             template<class T>
             DateTimeRange operator-(const DateTimeRange& r, Seconds<T> offset)
             {
                 DateTimeRange result{r};
                 shift(result,-offset);
                 return result;
             }
             const auto INVALID_RANGE
                 = DateTimeRange{std::numeric_limits<double>::quiet_NaN(), std::numeric_limits<double>::quiet_NaN()};
             inline QDebug operator<<(QDebug d, DateTimeRange obj)
             {
                 auto tendDateTimeStart = DateUtils::dateTime(obj.m_TStart);
                 auto tendDateTimeEnd = DateUtils::dateTime(obj.m_TEnd);
                 d << "ts: " << tendDateTimeStart << " te: " << tendDateTimeEnd;
                 return d;
             }
             DateTimeRange DateTimeRange::transform(const DateTimeRangeTransformation &tr) const noexcept
             {
                 return DateTimeRange{*this} * tr.zoom + tr.shift;
             }
             // Required for using shared_ptr in signals/slots
             SCIQLOP_REGISTER_META_TYPE(SQPRANGE_REGISTRY, DateTimeRange)
             #endif // SCIQLOP_DATETIMERANGE_H

include/Variable/Variable.h +7 -11

             #ifndef SCIQLOP_VARIABLE_H
             #define SCIQLOP_VARIABLE_H
             #include <QLoggingCategory>
             #include <QObject>
             #include <QUuid>
             #include <QReadWriteLock>
             #include "CoreGlobal.h"
             #include <Data/DataSeriesIterator.h>
             #include <Data/DataSeriesType.h>
             #include <Data/DateTimeRange.h>
             #include <Common/deprecate.h>
             #include <Common/MetaTypes.h>
             #include <Common/spimpl.h>
             Q_DECLARE_LOGGING_CATEGORY(LOG_Variable)
             class IDataSeries;
             class QString;
             /**
              * @brief The Variable class represents a variable in SciQlop.
              */
             class SCIQLOP_CORE_EXPORT Variable : public QObject {
                 Q_OBJECT
             public:
                 explicit Variable(const QString &name, const QVariantHash &metadata = {});
                 /// Copy ctor
                 explicit Variable(const Variable &other);
                 std::shared_ptr<Variable> clone() const;
                 QString name() const noexcept;
                 void setName(const QString &name) noexcept;
                 DateTimeRange range() const noexcept;
                 void setRange(const DateTimeRange &range, bool notify=false) noexcept;
                 DateTimeRange cacheRange() const noexcept;
                 void setCacheRange(const DateTimeRange &cacheRange) noexcept;
                 /// @return the number of points hold by the variable. The number of points is updated each time
                 /// the data series changes
                 unsigned int nbPoints() const noexcept;
                 /// Returns the real range of the variable, i.e. the min and max x-axis values of the data
                 /// series between the range of the variable. The real range is updated each time the variable
                 /// range or the data series changed
                 /// @return the real range, invalid range if the data series is null or empty
                 /// @sa setDataSeries()
                 /// @sa setRange()
                 std::optional<DateTimeRange> realRange() const noexcept;
                 /// @return the data of the variable, nullptr if there is no data
                 std::shared_ptr<IDataSeries> dataSeries() const noexcept;
                 /// @return the type of data that the variable holds
                 DataSeriesType type() const noexcept;
                 QVariantHash metadata() const noexcept;
+                void updateData(const std::vector<IDataSeries*>& dataSeries,
+                                const DateTimeRange& newRange, const DateTimeRange& newCacheRange,
+                                bool notify=true);
             DEPRECATE(
                 bool contains(const DateTimeRange &range) const noexcept;
                 bool intersect(const DateTimeRange &range) const noexcept;
                 bool isInside(const DateTimeRange &range) const noexcept;
                 bool cacheContains(const DateTimeRange &range) const noexcept;
                 bool cacheIntersect(const DateTimeRange &range) const noexcept;
                 bool cacheIsInside(const DateTimeRange &range) const noexcept;
-            DEPRECATE(
                 QVector<DateTimeRange> provideNotInCacheRangeList(const DateTimeRange &range) const noexcept;
                 QVector<DateTimeRange> provideInCacheRangeList(const DateTimeRange &range) const noexcept;
-            DEPRECATE(
                 void mergeDataSeries(std::shared_ptr<IDataSeries> dataSeries) noexcept;
-                void updateData(const std::vector<IDataSeries*>& dataSeries,
-                                const DateTimeRange& newRange, const DateTimeRange& newCacheRange,
-                                bool notify=true);
-            DEPRECATE(
                 static QVector<DateTimeRange> provideNotInCacheRangeList(const DateTimeRange &oldRange,
                                                                          const DateTimeRange &nextRange);
                 static QVector<DateTimeRange> provideInCacheRangeList(const DateTimeRange &oldRange,
                                                                       const DateTimeRange &nextRange);
                 )
+                operator QUuid() {return _uuid;}
                 QUuid ID(){return _uuid;}
             signals:
                 void updated();
                 DEPRECATE(
                         /// Signal emitted when when the data series of the variable is loaded for the first time
                         void dataInitialized();
                 )
                 private:
                     class VariablePrivate;
                 spimpl::unique_impl_ptr<VariablePrivate> impl;
                 QUuid _uuid;
                 QReadWriteLock m_lock;
             };
             // Required for using shared_ptr in signals/slots
             SCIQLOP_REGISTER_META_TYPE(VARIABLE_PTR_REGISTRY, std::shared_ptr<Variable>)
             SCIQLOP_REGISTER_META_TYPE(VARIABLE_PTR_VECTOR_REGISTRY, QVector<std::shared_ptr<Variable> >)
             #endif // SCIQLOP_VARIABLE_H

include/Variable/VariableController2.h +2 0

             #include <memory>
             #include <vector>
             #include <set>
             #include <QHash>
             #include <QObject>
             #include <QMutexLocker>
             #include <QUuid>
             #include <QItemSelectionModel>
             #include <Common/spimpl.h>
             #include <Variable/Variable.h>
             //#include <Variable/VariableSynchronizationGroup.h>
             #include <Variable/VariableModel.h>
             #include <Data/IDataProvider.h>
             #include "Data/DateTimeRange.h"
             class VariableController2: public QObject
             {
                 class VariableController2Private;
                 Q_OBJECT
                 spimpl::unique_impl_ptr<VariableController2Private> impl;
             public:
                 explicit VariableController2();
                 std::shared_ptr<Variable> createVariable(const QString &name, const QVariantHash &metadata,
                                                          const std::shared_ptr<IDataProvider>& provider,
                                                          const DateTimeRange &range);
                 void deleteVariable(const std::shared_ptr<Variable>& variable);
                 void changeRange(const std::shared_ptr<Variable>& variable, const DateTimeRange& r);
                 void asyncChangeRange(const std::shared_ptr<Variable>& variable, const DateTimeRange& r);
                 const std::set<std::shared_ptr<Variable>> variables();
+                bool isReady(const std::shared_ptr<Variable>& variable);
                 void synchronize(const std::shared_ptr<Variable>& var, const std::shared_ptr<Variable>& with);
             signals:
                 void variableAdded(const std::shared_ptr<Variable>&);
                 void variableDeleted(const std::shared_ptr<Variable>&);
             };

include/Variable/VariableSynchronizationGroup2.h +4 0

             #ifndef SCIQLOP_VARIABLESYNCHRONIZATIONGROUP2_H
             #define SCIQLOP_VARIABLESYNCHRONIZATIONGROUP2_H
             #include <QUuid>
             #include <set>
             #include "CoreGlobal.h"
             #include <Common/spimpl.h>
             #include <Common/containers.h>
             /**
              * @brief The VariableSynchronizationGroup2 class holds a list of Variables uuid which are synchronized
              * @note This class is part of SciQLop internals, as a normal user you shouldn't have to care about it
              */
             class SCIQLOP_CORE_EXPORT VariableSynchronizationGroup2
             {
             public:
                 explicit VariableSynchronizationGroup2()=default;
                 /**
                  * @brief VariableSynchronizationGroup2 is a convenience ctor to build a group with a default variable
                  * @param variable
                  */
                 explicit VariableSynchronizationGroup2(QUuid variable)
                     :_variables{{variable}}
                 {}
                 /**
                  * @brief addVariable adds the given variable to the group, does nothing if the varaible is alredy in the group
                  * @param variable
                  * @sa removeVariable
                  */
                 void addVariable(QUuid variable) noexcept
                 {
                     this->_variables.insert(variable);
                 }
                 /**
                  * @brief removeVariable removes the given variable from the group, does nothing if the varaible is not in the group
                  * @param variable
                  * @sa addVariable
                  */
                 void removeVariable(QUuid variable) noexcept
                 {
                     this->_variables.erase(variable);
                 }
                 /**
                  * @brief contains checks if the given variable is in the group
                  * @param variable
                  * @return true if the variable is in the group
                  */
                 bool contains(QUuid variable) const noexcept
                 {
                     return SciQLop::containers::contains(this->_variables,variable);
                 }
                 /**
                  * @brief variables
                  * @return the list of synchronized variables in this group as a std::set
                  */
                 const std::set<QUuid> &variables() const noexcept
                 {
                     return this->_variables;
                 }
+                inline QUuid ID(){return _ID;}
+                operator QUuid() {return _ID;}
             private:
                 std::set<QUuid> _variables;
+                QUuid _ID = QUuid::createUuid();
             };
             #endif // SCIQLOP_VARIABLESYNCHRONIZATIONGROUP2_H

src/Variable/VariableController2.cpp +218 -37

+            #include <QQueue>
+            #include <QThreadPool>
+            #include <QRunnable>
+            #include <QObject>
             #include "Variable/VariableController2.h"
             #include "Variable/VariableSynchronizationGroup2.h"
             #include <Common/containers.h>
             #include <Common/debug.h>
             #include <Data/DataProviderParameters.h>
             #include <Data/DateTimeRangeHelper.h>
+            #include <Data/DateTimeRange.h>
             #include <Variable/VariableCacheStrategyFactory.h>
+            #include <Variable/private/VCTransaction.h>
+            class Transactions
+            {
+                QReadWriteLock _mutex{QReadWriteLock::Recursive};
+                std::map<QUuid,std::optional<std::shared_ptr<VCTransaction>>> _nextTransactions;
+                std::map<QUuid,std::optional<std::shared_ptr<VCTransaction>>> _pendingTransactions;
+            public:
+                void addEntry(QUuid id)
+                {
+                    QWriteLocker lock{&_mutex};
+                    _nextTransactions[id] = std::nullopt;
+                    _pendingTransactions[id] = std::nullopt;
+                }
+                void removeEntry(QUuid id)
+                {
+                    QWriteLocker lock{&_mutex};
+                    _nextTransactions.erase(id);
+                    _pendingTransactions.erase(id);
+                }
+                std::map<QUuid,std::optional<std::shared_ptr<VCTransaction>>> pendingTransactions()
+                {
+                    QReadLocker lock{&_mutex};
+                    return _pendingTransactions;
+                }
+                std::map<QUuid,std::optional<std::shared_ptr<VCTransaction>>> nextTransactions()
+                {
+                    QReadLocker lock{&_mutex};
+                    return _nextTransactions;
+                }
+                std::optional<std::shared_ptr<VCTransaction>> start(QUuid id)
+                {
+                    QWriteLocker lock{&_mutex};
+                    _pendingTransactions[id] = _nextTransactions[id];
+                    _nextTransactions[id] = std::nullopt;
+                    return _pendingTransactions[id];
+                }
+                void enqueue(QUuid id, std::shared_ptr<VCTransaction> transaction)
+                {
+                    QWriteLocker lock{&_mutex};
+                    _nextTransactions[id] = transaction;
+                }
+                void complete(QUuid id)
+                {
+                    QWriteLocker lock{&_mutex};
+                    _pendingTransactions[id] = std::nullopt;
+                }
+                bool active(QUuid id)
+                {
+                    QReadLocker lock{&_mutex};
+                    return _nextTransactions[id].has_value() && _pendingTransactions[id].has_value();
+                }
+            };
             class VariableController2::VariableController2Private
             {
+                QThreadPool _ThreadPool;
                 QMap<QUuid,std::shared_ptr<Variable>> _variables;
                 QMap<QUuid,std::shared_ptr<IDataProvider>> _providers;
                 QMap<QUuid,std::shared_ptr<VariableSynchronizationGroup2>> _synchronizationGroups;
+                Transactions _transactions;
+                QReadWriteLock _lock{QReadWriteLock::Recursive};
                 std::unique_ptr<VariableCacheStrategy> _cacheStrategy;
                 bool p_contains(const std::shared_ptr<Variable>& variable)
                 {
-                    return _providers.contains(variable->ID());
+                    QReadLocker lock{&_lock};
+                    return _providers.contains(*variable);
                 }
                 bool v_contains(const std::shared_ptr<Variable>& variable)
                 {
+                    QReadLocker lock{&_lock};
                     return SciQLop::containers::contains(this->_variables, variable);
                 }
                 bool sg_contains(const std::shared_ptr<Variable>& variable)
                 {
-                    return _synchronizationGroups.contains(variable->ID());
+                    QReadLocker lock{&_lock};
+                    return _synchronizationGroups.contains(*variable);
                 }
-                void _changeRange(const std::shared_ptr<Variable>& var, DateTimeRange r)
+                void _transactionComplete(QUuid group, std::shared_ptr<VCTransaction> transaction)
+                {
+                    {
+                        QWriteLocker lock{&transaction->lock};
+                        transaction->ready -=1;
+                    }
+                    if(transaction->ready==0)
+                    {
+                        _transactions.complete(group);
+                    }
+                    this->_processTransactions();
+                }
+                void _processTransactions()
+                {
+                    QWriteLocker lock{&_lock};
+                    auto nextTransactions = _transactions.nextTransactions();
+                    auto pendingTransactions = _transactions.pendingTransactions();
+                    for( auto [groupID, newTransaction] : nextTransactions)
+                    {
+                        if(newTransaction.has_value() && !pendingTransactions[groupID].has_value())
+                        {
+                            _transactions.start(groupID);
+                            auto refVar = _variables[newTransaction.value()->refVar];
+                            auto ranges = _computeAllRangesInGroup(refVar,newTransaction.value()->range);
+                            for( auto const& [ID, range] : ranges)
+                            {
+                                auto provider = _providers[ID];
+                                auto variable = _variables[ID];
+                                auto [missingRanges, newCacheRange] = _computeMissingRanges(variable,range);
+                                auto exe = new TransactionExe(_variables[ID], provider, missingRanges, range, newCacheRange);
+                                QObject::connect(exe,
+                                        &TransactionExe::transactionComplete,
+                                        [groupID=groupID,transaction=newTransaction.value(),this]()
+                                        {
+                                            this->_transactionComplete(groupID, transaction);
+                                        }
+                                );
+                                _ThreadPool.start(exe);
+                            }
+                        }
+                    }
+                }
+                std::map<QUuid,DateTimeRange> _computeAllRangesInGroup(const std::shared_ptr<Variable>& refVar, DateTimeRange r)
+                {
+                    std::map<QUuid,DateTimeRange> ranges;
+                    if(!DateTimeRangeHelper::hasnan(r))
+                    {
+                        auto group = _synchronizationGroups[*refVar];
+                        if(auto transformation = DateTimeRangeHelper::computeTransformation(refVar->range(),r);
+                                transformation.has_value())
+                        {
+                            for(auto varId:group->variables())
+                            {
+                                auto var = _variables[varId];
+                                auto newRange = var->range().transform(transformation.value());
+                                ranges[varId] = newRange;
+                            }
+                        }
+                        else // force new range to all variables -> may be weird if more than one var in the group
+                            // @TODO ensure that there is no side effects
+                        {
+                            for(auto varId:group->variables())
+                            {
+                                auto var = _variables[varId];
+                                ranges[varId] = r;
+                            }
+                        }
+                    }
+                    else
+                    {
+                        SCIQLOP_ERROR(VariableController2Private, "Invalid range containing NaN");
+                    }
+                    return ranges;
+                }
+                std::pair<std::vector<DateTimeRange>,DateTimeRange> _computeMissingRanges(const std::shared_ptr<Variable>& var, DateTimeRange r)
                 {
-                    auto provider = _providers[var->ID()];
                     DateTimeRange newCacheRange;
                     std::vector<DateTimeRange> missingRanges;
                     if(DateTimeRangeHelper::hasnan(var->cacheRange()))
                     {
                         newCacheRange = _cacheStrategy->computeRange(r,r);
                         missingRanges = {newCacheRange};
                     }
                     else
                     {
                         newCacheRange = _cacheStrategy->computeRange(var->cacheRange(),r);
                         missingRanges = newCacheRange - var->cacheRange();
                     }
+                    return {missingRanges,newCacheRange};
+                }
+                void _changeRange(QUuid id, DateTimeRange r)
+                {
+                    _lock.lockForRead();
+                    auto var = _variables[id];
+                    _lock.unlock();
+                    _changeRange(var,r);
+                }
+                void _changeRange(const std::shared_ptr<Variable>& var, DateTimeRange r)
+                {
+                    auto provider = _providers[*var];
+                    auto [missingRanges, newCacheRange] = _computeMissingRanges(var,r);
                     std::vector<IDataSeries*> data;
                     for(auto range:missingRanges)
                     {
                         data.push_back(provider->getData(DataProviderParameters{{range}, var->metadata()}));
                     }
                     var->updateData(data, r, newCacheRange, true);
                 }
             public:
                 VariableController2Private(QObject* parent=Q_NULLPTR)
                     :_cacheStrategy(VariableCacheStrategyFactory::createCacheStrategy(CacheStrategy::SingleThreshold))
                 {
                     Q_UNUSED(parent);
+                    this->_ThreadPool.setMaxThreadCount(32);
                 }
-                ~VariableController2Private() = default;
+                ~VariableController2Private()
+                {
+                    while (this->_ThreadPool.activeThreadCount())
+                    {
+                        this->_ThreadPool.waitForDone(100);
+                    }
+                }
                 std::shared_ptr<Variable> createVariable(const QString &name, const QVariantHash &metadata, std::shared_ptr<IDataProvider> provider)
                 {
+                    QWriteLocker lock{&_lock};
                     auto newVar = std::make_shared<Variable>(name,metadata);
-                    this->_variables[newVar->ID()] = newVar;
+                    this->_variables[*newVar] = newVar;
-                    this->_providers[newVar->ID()] = std::move(provider);
+                    this->_providers[*newVar] = std::move(provider);
-                    this->_synchronizationGroups[newVar->ID()] = std::make_shared<VariableSynchronizationGroup2>(newVar->ID());
+                    auto group = std::make_shared<VariableSynchronizationGroup2>(newVar->ID());
+                    this->_synchronizationGroups[*newVar] = group;
+                    this->_transactions.addEntry(*group);
                     return newVar;
                 }
+                bool hasPendingTransactions(const std::shared_ptr<Variable>& variable)
+                {
+                    QReadLocker lock{&_lock};
+                    auto group = _synchronizationGroups[*variable];
+                    return _transactions.active(*group);
+                }
                 void deleteVariable(const std::shared_ptr<Variable>& variable)
                 {
                     /*
                      * Removing twice a var is ok but a var without provider has to be a hard error
                      * this means we got the var controller in an inconsistent state
                      */
                     if(v_contains(variable))
-                        this->_variables.remove(variable->ID());
+                        QWriteLocker lock{&_lock};
+                        this->_variables.remove(*variable);
+                    }
                     if(p_contains(variable))
-                        this->_providers.remove(variable->ID());
+                        QWriteLocker lock{&_lock};
+                        this->_providers.remove(*variable);
+                    }
                     else
                         SCIQLOP_ERROR(VariableController2Private, "No provider found for given variable");
                 }
                 void asyncChangeRange(const std::shared_ptr<Variable>& variable, const DateTimeRange& r)
                 {
-                void changeRange(const std::shared_ptr<Variable>& variable, DateTimeRange r)
                     if(p_contains(variable))
                     {
                         if(!DateTimeRangeHelper::hasnan(r))
                         {
-                            auto group = _synchronizationGroups[variable->ID()];
+                            auto group = _synchronizationGroups[*variable];
-                            if(auto transformation = DateTimeRangeHelper::computeTransformation(variable->range(),r);
+                            // Just overwrite next transaction
-                                    transformation.has_value())
-                                for(auto varId:group->variables())
-                                    auto var = _variables[varId];
-                                    auto newRange = var->range().transform(transformation.value());
-                                    _changeRange(var,newRange);
-                            else // force new range to all variables -> may be weird if more than one var in the group
-                                // @TODO ensure that there is no side effects
-                                for(auto varId:group->variables())
                             {
-                                    auto var = _variables[varId];
+                                QWriteLocker lock{&_lock};
-                                    _changeRange(var,r);
+                                _transactions.enqueue(*group,std::make_shared<VCTransaction>(variable->ID(), r, static_cast<int>(group->variables().size())));
                             }
+                            _processTransactions();
                         }
                         else
                         {
                             SCIQLOP_ERROR(VariableController2Private, "Invalid range containing NaN");
                         }
                     }
                     else
                     {
                         SCIQLOP_ERROR(VariableController2Private, "No provider found for given variable");
                     }
                 }
+                void changeRange(const std::shared_ptr<Variable>& variable, DateTimeRange r)
+                {
+                    auto ranges = _computeAllRangesInGroup(variable,r);
+                    for( auto const& [ID, range] : ranges)
+                    {
+                        _changeRange(ID,range);
+                    }
+                }
                 void synchronize(const std::shared_ptr<Variable>& var, const std::shared_ptr<Variable>& with)
                 {
                     if(v_contains(var) && v_contains(with))
                     {
                         if(sg_contains(var) && sg_contains(with))
                         {
+                            QWriteLocker lock{&_lock};
                             auto dest_group = this->_synchronizationGroups[with->ID()];
-                            this->_synchronizationGroups[var->ID()] = dest_group;
+                            this->_synchronizationGroups[*var] = dest_group;
-                            dest_group->addVariable(var->ID());
+                            dest_group->addVariable(*var);
                         }
                         else
                         {
                             SCIQLOP_ERROR(VariableController2Private, "At least one of the given variables isn't in a sync group");
                         }
                     }
                     else
                     {
                         SCIQLOP_ERROR(VariableController2Private, "At least one of the given variables is not found");
                     }
                 }
                 const std::set<std::shared_ptr<Variable>> variables()
                 {
                     std::set<std::shared_ptr<Variable>> vars;
+                    QReadLocker lock{&_lock};
                     for(const auto  &var:_variables)
                     {
                         vars.insert(var);
                     }
                     return  vars;
                 }
             };
             VariableController2::VariableController2()
                 :impl{spimpl::make_unique_impl<VariableController2Private>()}
             {}
             std::shared_ptr<Variable> VariableController2::createVariable(const QString &name, const QVariantHash &metadata, const std::shared_ptr<IDataProvider>& provider, const DateTimeRange &range)
             {
                 auto var =  impl->createVariable(name, metadata, provider);
                 emit variableAdded(var);
                 if(!DateTimeRangeHelper::hasnan(range))
                     impl->changeRange(var,range);
                 else
                     SCIQLOP_ERROR(VariableController2, "Creating a variable with default constructed DateTimeRange is an error");
                 return var;
             }
             void VariableController2::deleteVariable(const std::shared_ptr<Variable>& variable)
             {
                 impl->deleteVariable(variable);
                 emit variableDeleted(variable);
             }
             void VariableController2::changeRange(const std::shared_ptr<Variable>& variable, const DateTimeRange& r)
             {
                 impl->changeRange(variable, r);
             }
             void VariableController2::asyncChangeRange(const std::shared_ptr<Variable> &variable, const DateTimeRange &r)
             {
                 impl->asyncChangeRange(variable, r);
             }
             const std::set<std::shared_ptr<Variable> > VariableController2::variables()
             {
                 return impl->variables();
             }
+            bool VariableController2::isReady(const std::shared_ptr<Variable> &variable)
+            {
+                return impl->hasPendingTransactions(variable);
+            }
             void VariableController2::synchronize(const std::shared_ptr<Variable> &var, const std::shared_ptr<Variable> &with)
             {
                 impl->synchronize(var, with);
             }

tests/CMakeLists.txt +1 0

             add_definitions(-DCORE_TESTS_RESOURCES_DIR="${CMAKE_CURRENT_LIST_DIR}/../tests-resources")
             FILE (GLOB_RECURSE  TestUtilsSources
                 TestUtils/TestProviders.h
                 TestUtils/TestProviders.cpp
             )
             add_library(TestUtils ${TestUtilsSources})
             target_link_libraries(TestUtils sciqlopcore Qt5::Test)
             declare_test(TestStringUtils TestStringUtils Common/TestStringUtils.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestContainers TestContainers Common/TestContainers.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestSyncGroup TestSyncGroup Variable/TestSyncGroup.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestDateTimeRange TestDateTimeRange Data/TestDateTimeRange.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestDataSeriesUtils TestDataSeriesUtils Data/TestDataSeriesUtils.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestOptionalAxis TestOptionalAxis Data/TestOptionalAxis.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestSpectrogramSeries TestSpectrogramSeries
                 "Data/TestSpectrogramSeries.cpp;Data/DataSeriesBuilders.h;Data/DataSeriesBuilders.cpp;Data/DataSeriesTestsUtils.h;Data/DataSeriesTestsUtils.cpp"
                 "sciqlopcore;Qt5::Test")
             declare_test(TestOneDimArrayData TestOneDimArrayData Data/TestOneDimArrayData.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestScalarSeries TestScalarSeries
                 "Data/TestScalarSeries.cpp;Data/DataSeriesBuilders.h;Data/DataSeriesBuilders.cpp;Data/DataSeriesTestsUtils.h;Data/DataSeriesTestsUtils.cpp"
                 "sciqlopcore;Qt5::Test")
             declare_test(TestTwoDimArrayData TestTwoDimArrayData Data/TestTwoDimArrayData.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestVectorSeries TestVectorSeries
                 "Data/TestVectorSeries.cpp;Data/DataSeriesBuilders.h;Data/DataSeriesBuilders.cpp;Data/DataSeriesTestsUtils.h;Data/DataSeriesTestsUtils.cpp"
                 "sciqlopcore;Qt5::Test")
             declare_test(TestDataSourceController TestDataSourceController
                 "DataSource/TestDataSourceController.cpp;DataSource/DataSourceItemBuilder.cpp"
                 "sciqlopcore;Qt5::Test")
             declare_test(TestDataSourceItem TestDataSourceItem
                 "DataSource/TestDataSourceItem.cpp;DataSource/DataSourceItemBuilder.cpp"
                 "sciqlopcore;Qt5::Test")
             declare_test(TestVariable TestVariable Variable/TestVariable.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestVariableCacheController TestVariableCacheController Variable/TestVariableCacheController.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestVariableController TestVariableController Variable/TestVariableController.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestVariableSync TestVariableSync Variable/TestVariableSync.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestDownloader TestDownloader Network/TestDownloader.cpp "sciqlopcore;Qt5::Test")
             declare_test(TestVariableController2 TestVariableController2 Variable/TestVariableController2.cpp "sciqlopcore;TestUtils;Qt5::Test")
+            declare_test(TestVariableController2Async TestVariableController2Async Variable/TestVariableController2Async.cpp "sciqlopcore;TestUtils;Qt5::Test")
             declare_test(TestVariableController2WithSync TestVariableController2WithSync Variable/TestVariableController2WithSync.cpp "sciqlopcore;TestUtils;Qt5::Test")

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages