From 78e9c85d13ee930a93bc9bf6a898181cd625fa4e 2019-03-30 15:13:59 From: Alexis Jeandet Date: 2019-03-30 15:13:59 Subject: [PATCH] Some more progress on Python Providers side, works with a simple request on CDAWeb. Signed-off-by: Alexis Jeandet --- diff --git a/core b/core index 68c0115..b9379e0 160000 --- a/core +++ b/core @@ -1 +1 @@ -Subproject commit 68c01155acd227b01fa3c9f7926215e8cedd70df +Subproject commit b9379e008899fce30581fbbf2b3fefd6fe36cee2 diff --git a/plugins/python_providers/include/python_interpreter.h b/plugins/python_providers/include/python_interpreter.h index 9ce9a96..92e40f7 100644 --- a/plugins/python_providers/include/python_interpreter.h +++ b/plugins/python_providers/include/python_interpreter.h @@ -8,12 +8,14 @@ class PythonInterpreter { public: + using provider_funct_t = std::function(std::string&, double, double)>; PythonInterpreter(); - void add_register_callback(std::function&, - std::function(std::string&, double, double)>)> + void add_register_callback(std::function>>>&, + provider_funct_t)> callback); ~PythonInterpreter(); void eval(const std::string& file); + void release(); private: }; diff --git a/plugins/python_providers/include/python_providers.h b/plugins/python_providers/include/python_providers.h index 42afa5d..e910000 100644 --- a/plugins/python_providers/include/python_providers.h +++ b/plugins/python_providers/include/python_providers.h @@ -24,8 +24,8 @@ public: ~PythonProviders(); private: - void register_product(const std::vector& path_list, - std::function(std::string& name, double, double)> + void register_product(const std::vector>>>& product_list, + PythonInterpreter::provider_funct_t f); PythonInterpreter _interpreter; }; diff --git a/plugins/python_providers/resources/amda.py b/plugins/python_providers/resources/amda.py new file mode 100644 index 0000000..25e9f50 --- /dev/null +++ b/plugins/python_providers/resources/amda.py @@ -0,0 +1,28 @@ +import sys +sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core") +import os +import datetime +import PythonProviders +import pysciqlopcore +import numpy as np +import pandas as pds +import requests + +def get_sample(name,start,stop): + try: + tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ') + tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ') + req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv" + resp = requests.get(req_url,headers={"Accept":"application/json"}) + csv_url = resp.json()['FileDescription'][0]['Name'] + df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True) + t = np.array([d.timestamp()-7200 for d in df.index]) + values = df.values + return pysciqlopcore.VectorTimeSerie(t,values) + except Exception as e: + print("fuck ",str(e)) + return pysciqlopcore.VectorTimeSerie(1) + +PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[("type","vector")])],get_sample) + + diff --git a/plugins/python_providers/src/python_interpreter.cpp b/plugins/python_providers/src/python_interpreter.cpp index 50429f6..3b5d812 100644 --- a/plugins/python_providers/src/python_interpreter.cpp +++ b/plugins/python_providers/src/python_interpreter.cpp @@ -18,8 +18,8 @@ PythonInterpreter::PythonInterpreter() py::initialize_interpreter(false); } -void PythonInterpreter::add_register_callback(std::function&, - std::function(std::string&, double, double)>)> +void PythonInterpreter::add_register_callback(std::function>>>&, + provider_funct_t)> callback) { py::module PythonProviders = py::module::import("PythonProviders"); @@ -43,5 +43,9 @@ void PythonInterpreter::eval(const std::string& file) { std::cout << pythonErr.what(); } +} + +void PythonInterpreter::release() +{ _rel = new py::gil_scoped_release(); } diff --git a/plugins/python_providers/src/python_providers.cpp b/plugins/python_providers/src/python_providers.cpp index 4de21d8..63957d7 100644 --- a/plugins/python_providers/src/python_providers.cpp +++ b/plugins/python_providers/src/python_providers.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -27,11 +28,7 @@ struct noop_deleter class PythonProvider : public IDataProvider { public: - PythonProvider( - std::function(std::string&, double, double)> f) - : _pythonFunction { f } - { - } + PythonProvider(PythonInterpreter::provider_funct_t f) : _pythonFunction { f } {} PythonProvider(const PythonProvider& other) : _pythonFunction { other._pythonFunction } {} @@ -44,33 +41,20 @@ public: auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString(); auto range = parameters.m_Range; auto result = _pythonFunction(product, range.m_TStart, range.m_TEnd); - if (auto ts = std::dynamic_pointer_cast(result)) - { - return new VectorTimeSerie(*ts); - } - if (auto ts = std::dynamic_pointer_cast(result)) - { - return new ScalarTimeSerie(*ts); - } - if (auto ts = std::dynamic_pointer_cast(result)) - { - return new SpectrogramTimeSerie(*ts); - } - return nullptr; + return TimeSeriesUtils::copy(result); } private: - std::function(std::string&, double, double)> - _pythonFunction; + PythonInterpreter::provider_funct_t _pythonFunction; }; void PythonProviders::initialize() { _interpreter.add_register_callback( - [this](const std::vector& path_list, - std::function(std::string&, double, double)> - f) { this->register_product(path_list, f); }); + [this](const std::vector>>>& product_list, + PythonInterpreter::provider_funct_t f) { this->register_product(product_list, f); }); for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation)) { @@ -87,6 +71,7 @@ void PythonProviders::initialize() } } } + _interpreter.release(); } PythonProviders::~PythonProviders() {} @@ -137,20 +122,28 @@ std::unique_ptr make_product_item( return result; } -void PythonProviders::register_product(const std::vector& path_list, - std::function(std::string&, double, double)> f) +void PythonProviders::register_product( + const std::vector>>>& + product_list, + PythonInterpreter::provider_funct_t f) { auto& dataSourceController = sqpApp->dataSourceController(); auto id = dataSourceController.registerDataSource(DATA_SOURCE_NAME); auto root = make_folder_item(DATA_SOURCE_NAME); - std::for_each( - std::cbegin(path_list), std::cend(path_list), [id, f, root = root.get()](const auto& path) { + std::for_each(std::cbegin(product_list), std::cend(product_list), + [id, f, root = root.get()](const auto& product) { + const auto& path = product.first; auto path_list = QString::fromStdString(path).split('/'); auto name = *(std::cend(path_list) - 1); auto path_item = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root); - path_item->appendChild( - make_product_item({ { DataSourceItem::NAME_DATA_KEY, name } }, id)); + QVariantHash metaData { { DataSourceItem::NAME_DATA_KEY, name } }; + std::for_each(std::cbegin(product.second), std::cend(product.second), + [&metaData](const auto& mdata) { + metaData[QString::fromStdString(mdata.first)] + = QString::fromStdString(mdata.second); + }); + path_item->appendChild(make_product_item(metaData, id)); }); dataSourceController.setDataSourceItem(id, std::move(root)); dataSourceController.setDataProvider(id, std::make_unique(f));