From 704e0f1deb02fbfd8f44e58422a4b86f372e50e1 2019-08-24 12:34:58 From: Alexis Jeandet Date: 2019-08-24 12:34:58 Subject: [PATCH] Wrapper prototype working, with all kind of Serie Signed-off-by: Alexis Jeandet --- diff --git a/app/PySide2-bindings/CMakeLists.txt b/app/PySide2-bindings/CMakeLists.txt index 8ee7931..500d5bf 100644 --- a/app/PySide2-bindings/CMakeLists.txt +++ b/app/PySide2-bindings/CMakeLists.txt @@ -10,6 +10,7 @@ set(BINDINGS_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}") configure_file("${BINDINGS_SRC_DIR}/bindings.xml" "${BINDINGS_BUILD_DIR}/bindings.xml" COPYONLY) configure_file("${BINDINGS_SRC_DIR}/main.py" "${BINDINGS_BUILD_DIR}/main.py" COPYONLY) +configure_file("${BINDINGS_SRC_DIR}/TestPlugin.py" "${BINDINGS_BUILD_DIR}/plugins/TestPlugin.py" COPYONLY) execute_process(COMMAND "${PYTHON_EXECUTABLE}" "${BINDINGS_SRC_DIR}/src_list.py" cmake "${BINDINGS_BUILD_DIR}" OUTPUT_VARIABLE BINDINGS_SOURCE) diff --git a/app/PySide2-bindings/PyDataProvider.h b/app/PySide2-bindings/PyDataProvider.h index d62c84b..31e8993 100644 --- a/app/PySide2-bindings/PyDataProvider.h +++ b/app/PySide2-bindings/PyDataProvider.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -20,7 +21,7 @@ struct Product : path { path }, components { components }, metadata { metadata } { } - virtual ~Product() = default; + ~Product() = default; }; class PyDataProvider : public IDataProvider @@ -34,8 +35,7 @@ public: virtual ~PyDataProvider() {} - virtual QPair getData( - const std::string& key, double start_time, double stop_time) + virtual QPair,DataSeriesType> get_data(const QMap& key, double start_time, double stop_time) { (void)key, (void)start_time, (void)stop_time; return {}; @@ -45,10 +45,54 @@ public: { if (parameters.m_Data.contains("name")) { - auto data = getData(parameters.m_Data["name"].toString().toStdString(), + QMap metadata; + std::for_each(parameters.m_Data.constKeyValueBegin(), parameters.m_Data.constKeyValueEnd(), [&metadata](const auto& item) { + metadata[item.first] = item.second.toString(); + }); + auto [data, type] = get_data(metadata, parameters.m_Range.m_TStart, parameters.m_Range.m_TEnd); // TODO add shape/type switch - return new ScalarTimeSerie { data.first.to_std_vect(), data.second.to_std_vect() }; + //if (builder) + { + auto& [t,y]=data; + switch (type) + { + case DataSeriesType::SCALAR: + return new ScalarTimeSerie { std::move(t.data), + std::move(y.data) }; + break; + case DataSeriesType::VECTOR: + return new VectorTimeSerie { std::move(t.data), + y.to_std_vect_vect() }; + break; + case DataSeriesType::MULTICOMPONENT: + { + auto y_size = y.flat_size(); + auto t_size = t.flat_size(); + + if(t_size && (y_size%t_size)==0) + { + return new MultiComponentTimeSerie { std::move(t.data), + std::move(y.data),{t_size, y_size/t_size} }; + } + break; + } + case DataSeriesType::SPECTROGRAM: + { + auto y_size = y.flat_size(); + auto t_size = t.flat_size(); + + if(t_size && (y_size%t_size)==0) + { + return new SpectrogramTimeSerie { std::move(t.data), + std::move(y.data),{t_size, y_size/t_size} }; + } + break; + } + default: + break; + } + } } return nullptr; } @@ -65,25 +109,3 @@ public: }); } }; - - -struct Providers -{ - Providers() = default; - virtual ~Providers() = default; - inline void register_provider(PyDataProvider* provider) - { - auto& dataSourceController = sqpApp->dataSourceController(); - dataSourceController.setDataProvider( - provider->id(), std::unique_ptr(provider)); - } -}; - - -inline ScalarTimeSerie test_PyDataProvider(PyDataProvider& prov) -{ - auto v = prov.getData("", 0., 0.); - ScalarTimeSerie s; - s.set_data(v.first.to_std_vect(), v.second.to_std_vect()); - return s; -} diff --git a/app/PySide2-bindings/TestPlugin.py b/app/PySide2-bindings/TestPlugin.py new file mode 100644 index 0000000..f959b0b --- /dev/null +++ b/app/PySide2-bindings/TestPlugin.py @@ -0,0 +1,94 @@ +import traceback +from SciQLopBindings import PyDataProvider, Product, VectorTimeSerie, ScalarTimeSerie, DataSeriesType +import numpy as np +import math +from spwc.cache import _cache +from spwc.common.datetime_range import DateTimeRange +from functools import partial +from datetime import datetime, timedelta, timezone +from spwc.common.variable import SpwcVariable + + +def make_scalar(x): + y = np.cos(x/10.) + return SpwcVariable(time=x, data=y) + +def make_vector(x): + v=np.ones((len(x),3)) + for i in range(3): + v.transpose()[:][i] = np.cos(x/10. + float(i)) + (100. * np.cos(x/10000. + float(i))) + return SpwcVariable(time=x, data=v) + + +def make_multicomponent(x): + v=np.ones((len(x),4)) + for i in range(4): + v.transpose()[:][i] = float(i+1) * np.cos(x/10. + float(i)) + return SpwcVariable(time=x, data=v) + +def make_spectrogram(x): + v=np.ones((len(x),32)) + for i in range(32): + v.transpose()[:][i] = 100.*(2.+ float(i+1) * np.cos(x/1024. + float(i))) + return SpwcVariable(time=x, data=v) + + +def _get_data(p_type, start, stop): + if type(start) is datetime: + start = start.timestamp() + stop = stop.timestamp() + x = np.arange(math.ceil(start), math.floor(stop))*1. + if p_type == 'scalar': + return make_scalar(x) + if p_type == 'vector': + return make_vector(x) + if p_type == 'multicomponent': + return make_multicomponent(x) + if p_type == 'spectrogram': + return make_spectrogram(np.arange(math.ceil(start), math.floor(stop),15.)) + return None + +class MyProvider(PyDataProvider): + def __init__(self): + super(MyProvider,self).__init__() + self.register_products([Product("/tests/without_cache/scalar",[],{"type":"scalar"}), + Product("/tests/without_cache/vector",[],{"type":"vector"}), + Product("/tests/without_cache/multicomponent",[],{"type":"multicomponent",'size':'4'}), + Product("/tests/without_cache/spectrogram",[],{"type":"spectrogram",'size':'32'}), + Product("/tests/with_cache/scalar",[],{"type":"scalar", "cache":"true"}), + Product("/tests/with_cache/vector",[],{"type":"vector", "cache":"true"}), + Product("/tests/with_cache/multicomponent",[],{"type":"multicomponent",'size':'4', "cache":"true"}) + ]) + + def get_data(self,metadata,start,stop): + ts_type = DataSeriesType.SCALAR + default_ctor_args = 1 + use_cache = False + p_type = 'scalar' + try: + for key,value in metadata.items(): + if key == 'type': + p_type = value + if value == 'vector': + ts_type = DataSeriesType.VECTOR + elif value == 'multicomponent': + ts_type = DataSeriesType.MULTICOMPONENT + elif value == 'spectrogram': + ts_type = DataSeriesType.SPECTROGRAM + if key == 'cache' and value == 'true': + use_cache = True + if use_cache: + cache_product = f"tests/{p_type}" + var = _cache.get_data(cache_product, DateTimeRange(datetime.fromtimestamp(start, tz=timezone.utc), datetime.fromtimestamp(stop, tz=timezone.utc)), partial(_get_data, p_type), fragment_hours=24) + else: + print("No Cache") + var = _get_data(p_type, start, stop) + return ((var.time,var.data), ts_type) + except Exception as e: + print(traceback.format_exc()) + print("Error in test.py ",str(e)) + return ((np.array(), np.array()), ts_type) + + +t=MyProvider() + diff --git a/app/PySide2-bindings/bindings.h b/app/PySide2-bindings/bindings.h index 5644a3c..41c26ea 100644 --- a/app/PySide2-bindings/bindings.h +++ b/app/PySide2-bindings/bindings.h @@ -6,6 +6,8 @@ #include "numpy_wrappers.h" #include #include +#include +#include #include diff --git a/app/PySide2-bindings/bindings.xml b/app/PySide2-bindings/bindings.xml index 70bac73..d6b8db7 100644 --- a/app/PySide2-bindings/bindings.xml +++ b/app/PySide2-bindings/bindings.xml @@ -5,6 +5,7 @@ + @@ -31,19 +32,26 @@ - + + + + + auto result = %in.py_object(); + return result; + + + + %out = %OUTTYPE(%in); + + + + + - - - - - - - @@ -55,27 +63,29 @@ - - - - - auto result = %in.py_object(); - return result; - - - - %out = %OUTTYPE(%in); - - - - %BEGIN_ALLOW_THREADS %0 = new ScalarTimeSerieWrapper(); - %0.set_data(%1.to_std_vect(),%2.to_std_vect()); + %0.set_data(std::move(%1.data),std::move(%2.data)); + %END_ALLOW_THREADS + + + + + %RETURN_TYPE %0 = %CPPSELF.%FUNCTION_NAME(); + %PYARG_0 = %CONVERTTOPYTHON[%RETURN_TYPE](%0); + + + + + + + %BEGIN_ALLOW_THREADS + %0 = new VectorTimeSerieWrapper(); + %0.set_data(std::move(%1.data),%2.to_std_vect_vect()); %END_ALLOW_THREADS @@ -86,6 +96,8 @@ - + + + diff --git a/app/PySide2-bindings/main.py b/app/PySide2-bindings/main.py index 7bd6566..ce03c19 100644 --- a/app/PySide2-bindings/main.py +++ b/app/PySide2-bindings/main.py @@ -99,5 +99,8 @@ if __name__ == "__main__": term = IPythonDockWidget(available_vars={"app":app, "main_window":main_window}, custom_banner="SciQLop IPython Console ") main_window.addDockWidget(Qt.BottomDockWidgetArea, term) main_window.show() + for file in os.listdir('plugins'): + if os.path.isfile(f"plugins/{file}"): + exec(open(f"plugins/{file}").read()) sys.exit(app.exec_()) diff --git a/app/PySide2-bindings/numpy_wrappers.h b/app/PySide2-bindings/numpy_wrappers.h index 63feb93..64cbfe7 100644 --- a/app/PySide2-bindings/numpy_wrappers.h +++ b/app/PySide2-bindings/numpy_wrappers.h @@ -1,6 +1,7 @@ #ifndef NUMPY_WRAPPERS_H #define NUMPY_WRAPPERS_H #include +#include #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #if defined(slots) && (defined(__GNUC__) || defined(_MSC_VER) || defined(__clang__)) #pragma push_macro("slots") @@ -42,10 +43,7 @@ struct PyObjectWrapper { private: PyObject* _py_obj = nullptr; - void inc_refcount() - { - Py_XINCREF(_py_obj); - } + void inc_refcount() { Py_XINCREF(_py_obj); } void dec_refcount() { Py_XDECREF(_py_obj); @@ -56,10 +54,7 @@ public: PyObjectWrapper() : _py_obj { nullptr } {} PyObjectWrapper(const PyObjectWrapper& other) : _py_obj { other._py_obj } { inc_refcount(); } PyObjectWrapper(PyObjectWrapper&& other) : _py_obj { other._py_obj } { inc_refcount(); } - explicit PyObjectWrapper(PyObject* obj) : _py_obj { obj } - { - inc_refcount(); - } + explicit PyObjectWrapper(PyObject* obj) : _py_obj { obj } { inc_refcount(); } ~PyObjectWrapper() { dec_refcount(); } PyObjectWrapper& operator=(PyObjectWrapper&& other) { @@ -81,13 +76,11 @@ public: inline bool is_null() { return _py_obj == nullptr; } }; -struct NpArray +struct NpArray_view { private: PyObjectWrapper _py_obj; - NpArray(NpArray& other) = delete; - NpArray(const NpArray& other) = delete; - NpArray(const NpArray&& other) = delete; + NpArray_view(const NpArray_view&& other) = delete; public: static bool isNpArray(PyObject* obj) @@ -96,21 +89,22 @@ public: auto is_c_aray = obj && PyArray_Check(arr) && PyArray_ISCARRAY(arr); return is_c_aray; } - NpArray() : _py_obj { nullptr } {} - NpArray(NpArray&& other) : _py_obj { other._py_obj } {} - explicit NpArray(PyObject* obj) : _py_obj { obj } + NpArray_view() : _py_obj { nullptr } {} + NpArray_view(const NpArray_view& other) : _py_obj { other._py_obj } {} + NpArray_view(NpArray_view&& other) : _py_obj { other._py_obj } {} + explicit NpArray_view(PyObject* obj) : _py_obj { obj } { assert(isNpArray(obj)); assert(PyArray_ISFLOAT(_py_obj.get())); } - NpArray& operator=(const NpArray& other) + NpArray_view& operator=(const NpArray_view& other) { this->_py_obj = other._py_obj; return *this; } - NpArray& operator=(NpArray&& other) + NpArray_view& operator=(NpArray_view&& other) { this->_py_obj = other._py_obj; return *this; @@ -133,10 +127,32 @@ public: return shape; } + std::size_t ndim() + { + if (!_py_obj.is_null()) + { + return static_cast(PyArray_NDIM(_py_obj.get())); + } + return 0; + } + + std::size_t size(std::size_t index = 0) + { + if (!_py_obj.is_null()) + { + if (index < static_cast(PyArray_NDIM(_py_obj.get()))) + { + return PyArray_SHAPE(_py_obj.get())[index]; + } + } + return 0; + } + std::size_t flat_size() { auto s = this->shape(); - return std::accumulate(std::cbegin(s), std::cend(s), 0); + return std::accumulate( + std::cbegin(s), std::cend(s), 1, [](const auto& a, const auto& b) { return a * b; }); } double data(std::size_t pos) @@ -150,6 +166,7 @@ public: std::vector to_std_vect() { + assert(!this->_py_obj.is_null()); auto sz = flat_size(); std::vector v(sz); auto d_ptr = reinterpret_cast(PyArray_DATA(_py_obj.get())); @@ -157,12 +174,77 @@ public: return v; } + std::vector to_std_vect_vect() + { + auto sz = size(0); + std::vector v(sz); + if (sz) + { + assert(ndim() == 2); + assert(size(1) == 3); + auto d_ptr + = reinterpret_cast(PyArray_DATA(_py_obj.get())); + std::copy(d_ptr, d_ptr + sz, std::begin(v)); + } + return v; + } + PyObject* py_object() { return _py_obj.py_object(); } }; +struct NpArray +{ + std::vector shape; + std::vector data; + static bool isNpArray(PyObject* obj) { return NpArray_view::isNpArray(obj); } + NpArray() = default; + explicit NpArray(PyObject* obj) + { + if (obj) + { + NpArray_view view { obj }; + shape = view.shape(); + data = view.to_std_vect(); + } + } + + inline std::size_t ndim() { return shape.size(); } + + std::size_t size(std::size_t index = 0) + { + if (index < shape.size()) + return shape[index]; + return 0; + } + + std::size_t flat_size() + { + return std::accumulate(std::cbegin(shape), std::cend(shape), 1, + [](const auto& a, const auto& b) { return a * b; }); + } + + // TODO temporary hack should find a way to avoid this copy + std::vector to_std_vect_vect() + { + auto sz = size(0); + std::vector v(sz); + if (sz) + { + assert(ndim() == 2); + assert(size(1) == 3); + auto d_ptr = reinterpret_cast(data.data()); + std::copy(d_ptr, d_ptr + sz, std::begin(v)); + } + return v; + } + + // TODO maybe ;) + PyObject* py_object() { return nullptr; } +}; + inline int test_np_array(NpArray& arr) { - auto shape = arr.shape(); + auto shape = arr.shape; std::cout << "len(shape)=" << shape.size() << std::endl; std::for_each(std::cbegin(shape), std::cend(shape), [](auto sz) { static int i = 0; @@ -171,7 +253,7 @@ inline int test_np_array(NpArray& arr) auto flatsize = std::accumulate(std::cbegin(shape), std::cend(shape), 0); for (auto i = 0; i < flatsize; i++) { - std::cout << "data[" << i << "]=" << arr.data(i) << std::endl; + std::cout << "data[" << i << "]=" << arr.data[i] << std::endl; } return 1; }