##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Some more progress on Python Providers side, works with a simple request on CDAWeb....
jeandet -
r1429:78e9c85d13ee
parent child
Show More
Side by Side Unified
Context file:
r1429:78e9c85d13ee Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,28
1 import sys
2 sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")
3 import os
4 import datetime
5 import PythonProviders
6 import pysciqlopcore
7 import numpy as np
8 import pandas as pds
9 import requests
10
11 def get_sample(name,start,stop):
12 try:
13 tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ')
14 tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ')
15 req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv"
16 resp = requests.get(req_url,headers={"Accept":"application/json"})
17 csv_url = resp.json()['FileDescription'][0]['Name']
18 df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True)
19 t = np.array([d.timestamp()-7200 for d in df.index])
20 values = df.values
21 return pysciqlopcore.VectorTimeSerie(t,values)
22 except Exception as e:
23 print("fuck ",str(e))
24 return pysciqlopcore.VectorTimeSerie(1)
25
26 PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[("type","vector")])],get_sample)
27
28
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 Subproject commit 68c01155acd227b01fa3c9f7926215e8cedd70df
1 Subproject commit b9379e008899fce30581fbbf2b3fefd6fe36cee2
Show file before | Show file after | Hide comments
@@ -1,19 +1,21
1 #include <Data/DateTimeRange.h>
1 #include <Data/DateTimeRange.h>
2 #include <TimeSeries.h>
2 #include <TimeSeries.h>
3 #include <functional>
3 #include <functional>
4 #include <iostream>
4 #include <iostream>
5 #include <memory>
5 #include <memory>
6
6
7
7
8 class PythonInterpreter
8 class PythonInterpreter
9 {
9 {
10 public:
10 public:
11 using provider_funct_t = std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>;
11 PythonInterpreter();
12 PythonInterpreter();
12 void add_register_callback(std::function<void(const std::vector<std::string>&,
13 void add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
13 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>)>
14 provider_funct_t)>
14 callback);
15 callback);
15 ~PythonInterpreter();
16 ~PythonInterpreter();
16 void eval(const std::string& file);
17 void eval(const std::string& file);
18 void release();
17
19
18 private:
20 private:
19 };
21 };
Show file before | Show file after | Hide comments
@@ -1,33 +1,33
1 #ifndef PYTHON_PROVIDERS_H
1 #ifndef PYTHON_PROVIDERS_H
2 #define PYTHON_PROVIDERS_H
2 #define PYTHON_PROVIDERS_H
3
3
4 #include <Plugin/IPlugin.h>
4 #include <Plugin/IPlugin.h>
5 #include <QUuid>
5 #include <QUuid>
6
6
7 #include <memory>
7 #include <memory>
8 #include <python_interpreter.h>
8 #include <python_interpreter.h>
9
9
10 #ifndef SCIQLOP_PLUGIN_JSON_FILE_PATH
10 #ifndef SCIQLOP_PLUGIN_JSON_FILE_PATH
11 #define SCIQLOP_PLUGIN_JSON_FILE_PATH "python_providers.json"
11 #define SCIQLOP_PLUGIN_JSON_FILE_PATH "python_providers.json"
12 #endif
12 #endif
13
13
14 class DataSourceItem;
14 class DataSourceItem;
15
15
16 class PythonProviders : public QObject, public IPlugin
16 class PythonProviders : public QObject, public IPlugin
17 {
17 {
18 Q_OBJECT
18 Q_OBJECT
19 Q_INTERFACES(IPlugin)
19 Q_INTERFACES(IPlugin)
20 Q_PLUGIN_METADATA(IID "sciqlop.plugin.IPlugin" FILE SCIQLOP_PLUGIN_JSON_FILE_PATH)
20 Q_PLUGIN_METADATA(IID "sciqlop.plugin.IPlugin" FILE SCIQLOP_PLUGIN_JSON_FILE_PATH)
21 public:
21 public:
22 /// @sa IPlugin::initialize()
22 /// @sa IPlugin::initialize()
23 void initialize() override;
23 void initialize() override;
24 ~PythonProviders();
24 ~PythonProviders();
25
25
26 private:
26 private:
27 void register_product(const std::vector<std::string>& path_list,
27 void register_product(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>& product_list,
28 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string& name, double, double)>
28 PythonInterpreter::provider_funct_t
29 f);
29 f);
30 PythonInterpreter _interpreter;
30 PythonInterpreter _interpreter;
31 };
31 };
32
32
33 #endif // PYTHON_PROVIDERS_H
33 #endif // PYTHON_PROVIDERS_H
Show file before | Show file after | Hide comments
@@ -1,47 +1,51
1 #include "python_interpreter.h"
1 #include "python_interpreter.h"
2 #include <Data/DateTimeRange.h>
2 #include <Data/DateTimeRange.h>
3 #include <TimeSeries.h>
3 #include <TimeSeries.h>
4 #include <functional>
4 #include <functional>
5 #include <iostream>
5 #include <iostream>
6 #include <pybind11/embed.h>
6 #include <pybind11/embed.h>
7 #include <pybind11/functional.h>
7 #include <pybind11/functional.h>
8 #include <pybind11/stl.h>
8 #include <pybind11/stl.h>
9
9
10 namespace py = pybind11;
10 namespace py = pybind11;
11
11
12
12
13 PYBIND11_EMBEDDED_MODULE(PythonProviders, m) {}
13 PYBIND11_EMBEDDED_MODULE(PythonProviders, m) {}
14 static pybind11::gil_scoped_release* _rel = nullptr;
14 static pybind11::gil_scoped_release* _rel = nullptr;
15
15
16 PythonInterpreter::PythonInterpreter()
16 PythonInterpreter::PythonInterpreter()
17 {
17 {
18 py::initialize_interpreter(false);
18 py::initialize_interpreter(false);
19 }
19 }
20
20
21 void PythonInterpreter::add_register_callback(std::function<void(const std::vector<std::string>&,
21 void PythonInterpreter::add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
22 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>)>
22 provider_funct_t)>
23 callback)
23 callback)
24 {
24 {
25 py::module PythonProviders = py::module::import("PythonProviders");
25 py::module PythonProviders = py::module::import("PythonProviders");
26 PythonProviders.attr("register_product") = callback;
26 PythonProviders.attr("register_product") = callback;
27 }
27 }
28
28
29 PythonInterpreter::~PythonInterpreter()
29 PythonInterpreter::~PythonInterpreter()
30 {
30 {
31 if (_rel)
31 if (_rel)
32 delete _rel;
32 delete _rel;
33 py::finalize_interpreter();
33 py::finalize_interpreter();
34 }
34 }
35
35
36 void PythonInterpreter::eval(const std::string& file)
36 void PythonInterpreter::eval(const std::string& file)
37 {
37 {
38 try
38 try
39 {
39 {
40 py::eval_file(file);
40 py::eval_file(file);
41 }
41 }
42 catch (py::error_already_set const& pythonErr)
42 catch (py::error_already_set const& pythonErr)
43 {
43 {
44 std::cout << pythonErr.what();
44 std::cout << pythonErr.what();
45 }
45 }
46 }
47
48 void PythonInterpreter::release()
49 {
46 _rel = new py::gil_scoped_release();
50 _rel = new py::gil_scoped_release();
47 }
51 }
Show file before | Show file after | Hide comments
@@ -1,158 +1,151
1 #include "python_providers.h"
1 #include "python_providers.h"
2 #include <Data/DataProviderParameters.h>
2 #include <Data/DataProviderParameters.h>
3 #include <Data/DateTimeRange.h>
3 #include <Data/DateTimeRange.h>
4 #include <Data/IDataProvider.h>
4 #include <Data/IDataProvider.h>
5 #include <Data/ScalarTimeSerie.h>
5 #include <Data/ScalarTimeSerie.h>
6 #include <Data/SpectrogramTimeSerie.h>
6 #include <Data/SpectrogramTimeSerie.h>
7 #include <Data/TimeSeriesUtils.h>
7 #include <Data/VectorTimeSerie.h>
8 #include <Data/VectorTimeSerie.h>
8 #include <DataSource/DataSourceController.h>
9 #include <DataSource/DataSourceController.h>
9 #include <DataSource/DataSourceItem.h>
10 #include <DataSource/DataSourceItem.h>
10 #include <DataSource/DataSourceItemAction.h>
11 #include <DataSource/DataSourceItemAction.h>
11 #include <QDir>
12 #include <QDir>
12 #include <QStandardPaths>
13 #include <QStandardPaths>
13 #include <QStringList>
14 #include <QStringList>
14 #include <SqpApplication.h>
15 #include <SqpApplication.h>
15 #include <TimeSeries.h>
16 #include <TimeSeries.h>
16 #include <functional>
17 #include <functional>
17 #include <iostream>
18 #include <iostream>
18
19
19
20
20 const auto DATA_SOURCE_NAME = QStringLiteral("PythonProviders");
21 const auto DATA_SOURCE_NAME = QStringLiteral("PythonProviders");
21
22
22 struct noop_deleter
23 struct noop_deleter
23 {
24 {
24 void operator()(TimeSeries::ITimeSerie*) {}
25 void operator()(TimeSeries::ITimeSerie*) {}
25 };
26 };
26
27
27 class PythonProvider : public IDataProvider
28 class PythonProvider : public IDataProvider
28 {
29 {
29 public:
30 public:
30 PythonProvider(
31 PythonProvider(PythonInterpreter::provider_funct_t f) : _pythonFunction { f } {}
31 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)> f)
32 : _pythonFunction { f }
33 {
34 }
35
32
36 PythonProvider(const PythonProvider& other) : _pythonFunction { other._pythonFunction } {}
33 PythonProvider(const PythonProvider& other) : _pythonFunction { other._pythonFunction } {}
37
34
38 std::shared_ptr<IDataProvider> clone() const override
35 std::shared_ptr<IDataProvider> clone() const override
39 {
36 {
40 return std::make_shared<PythonProvider>(*this);
37 return std::make_shared<PythonProvider>(*this);
41 }
38 }
42 virtual TimeSeries::ITimeSerie* getData(const DataProviderParameters& parameters) override
39 virtual TimeSeries::ITimeSerie* getData(const DataProviderParameters& parameters) override
43 {
40 {
44 auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString();
41 auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString();
45 auto range = parameters.m_Range;
42 auto range = parameters.m_Range;
46 auto result = _pythonFunction(product, range.m_TStart, range.m_TEnd);
43 auto result = _pythonFunction(product, range.m_TStart, range.m_TEnd);
47 if (auto ts = std::dynamic_pointer_cast<VectorTimeSerie>(result))
44 return TimeSeriesUtils::copy(result);
48 {
49 return new VectorTimeSerie(*ts);
50 }
51 if (auto ts = std::dynamic_pointer_cast<ScalarTimeSerie>(result))
52 {
53 return new ScalarTimeSerie(*ts);
54 }
55 if (auto ts = std::dynamic_pointer_cast<SpectrogramTimeSerie>(result))
56 {
57 return new SpectrogramTimeSerie(*ts);
58 }
59 return nullptr;
60 }
45 }
61
46
62 private:
47 private:
63 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>
48 PythonInterpreter::provider_funct_t _pythonFunction;
64 _pythonFunction;
65 };
49 };
66
50
67
51
68 void PythonProviders::initialize()
52 void PythonProviders::initialize()
69 {
53 {
70 _interpreter.add_register_callback(
54 _interpreter.add_register_callback(
71 [this](const std::vector<std::string>& path_list,
55 [this](const std::vector<std::pair<std::string,
72 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>
56 std::vector<std::pair<std::string, std::string>>>>& product_list,
73 f) { this->register_product(path_list, f); });
57 PythonInterpreter::provider_funct_t f) { this->register_product(product_list, f); });
74
58
75 for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation))
59 for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation))
76 {
60 {
77 auto dir = QDir(path + "/python");
61 auto dir = QDir(path + "/python");
78 if (dir.exists())
62 if (dir.exists())
79 {
63 {
80 for (const auto& entry :
64 for (const auto& entry :
81 dir.entryInfoList(QDir::Files | QDir::NoDotAndDotDot, QDir::Name))
65 dir.entryInfoList(QDir::Files | QDir::NoDotAndDotDot, QDir::Name))
82 {
66 {
83 if (entry.isFile() && entry.suffix() == "py")
67 if (entry.isFile() && entry.suffix() == "py")
84 {
68 {
85 _interpreter.eval(entry.absoluteFilePath().toStdString());
69 _interpreter.eval(entry.absoluteFilePath().toStdString());
86 }
70 }
87 }
71 }
88 }
72 }
89 }
73 }
74 _interpreter.release();
90 }
75 }
91
76
92 PythonProviders::~PythonProviders() {}
77 PythonProviders::~PythonProviders() {}
93
78
94 std::unique_ptr<DataSourceItem> make_folder_item(const QString& name)
79 std::unique_ptr<DataSourceItem> make_folder_item(const QString& name)
95 {
80 {
96 return std::make_unique<DataSourceItem>(DataSourceItemType::NODE, name);
81 return std::make_unique<DataSourceItem>(DataSourceItemType::NODE, name);
97 }
82 }
98
83
99 template <typename T>
84 template <typename T>
100 DataSourceItem* make_path_items(
85 DataSourceItem* make_path_items(
101 const T& path_list_begin, const T& path_list_end, DataSourceItem* root)
86 const T& path_list_begin, const T& path_list_end, DataSourceItem* root)
102 {
87 {
103 std::for_each(path_list_begin, path_list_end, [&root](const auto& folder_name) mutable {
88 std::for_each(path_list_begin, path_list_end, [&root](const auto& folder_name) mutable {
104 auto folder_ptr = root->findItem(folder_name);
89 auto folder_ptr = root->findItem(folder_name);
105 if (folder_ptr == nullptr)
90 if (folder_ptr == nullptr)
106 {
91 {
107 auto folder = make_folder_item(folder_name);
92 auto folder = make_folder_item(folder_name);
108 folder_ptr = folder.get();
93 folder_ptr = folder.get();
109 root->appendChild(std::move(folder));
94 root->appendChild(std::move(folder));
110 }
95 }
111 root = folder_ptr;
96 root = folder_ptr;
112 });
97 });
113 return root;
98 return root;
114 }
99 }
115
100
116 std::unique_ptr<DataSourceItem> make_product_item(
101 std::unique_ptr<DataSourceItem> make_product_item(
117 const QVariantHash& metaData, const QUuid& dataSourceUid)
102 const QVariantHash& metaData, const QUuid& dataSourceUid)
118 {
103 {
119 auto result = std::make_unique<DataSourceItem>(DataSourceItemType::PRODUCT, metaData);
104 auto result = std::make_unique<DataSourceItem>(DataSourceItemType::PRODUCT, metaData);
120
105
121 // Adds plugin name to product metadata
106 // Adds plugin name to product metadata
122 result->setData(DataSourceItem::PLUGIN_DATA_KEY, DATA_SOURCE_NAME);
107 result->setData(DataSourceItem::PLUGIN_DATA_KEY, DATA_SOURCE_NAME);
123 result->setData(DataSourceItem::ID_DATA_KEY, metaData.value(DataSourceItem::NAME_DATA_KEY));
108 result->setData(DataSourceItem::ID_DATA_KEY, metaData.value(DataSourceItem::NAME_DATA_KEY));
124
109
125 auto productName = metaData.value(DataSourceItem::NAME_DATA_KEY).toString();
110 auto productName = metaData.value(DataSourceItem::NAME_DATA_KEY).toString();
126
111
127 // Add action to load product from DataSourceController
112 // Add action to load product from DataSourceController
128 result->addAction(
113 result->addAction(
129 std::make_unique<DataSourceItemAction>(QObject::tr("Load %1 product").arg(productName),
114 std::make_unique<DataSourceItemAction>(QObject::tr("Load %1 product").arg(productName),
130 [productName, dataSourceUid](DataSourceItem& item) {
115 [productName, dataSourceUid](DataSourceItem& item) {
131 if (auto app = sqpApp)
116 if (auto app = sqpApp)
132 {
117 {
133 app->dataSourceController().loadProductItem(dataSourceUid, item);
118 app->dataSourceController().loadProductItem(dataSourceUid, item);
134 }
119 }
135 }));
120 }));
136
121
137 return result;
122 return result;
138 }
123 }
139
124
140 void PythonProviders::register_product(const std::vector<std::string>& path_list,
125 void PythonProviders::register_product(
141 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)> f)
126 const std::vector<std::pair<std::string, std::vector<std::pair<std::string, std::string>>>>&
127 product_list,
128 PythonInterpreter::provider_funct_t f)
142 {
129 {
143 auto& dataSourceController = sqpApp->dataSourceController();
130 auto& dataSourceController = sqpApp->dataSourceController();
144 auto id = dataSourceController.registerDataSource(DATA_SOURCE_NAME);
131 auto id = dataSourceController.registerDataSource(DATA_SOURCE_NAME);
145 auto root = make_folder_item(DATA_SOURCE_NAME);
132 auto root = make_folder_item(DATA_SOURCE_NAME);
146 std::for_each(
133 std::for_each(std::cbegin(product_list), std::cend(product_list),
147 std::cbegin(path_list), std::cend(path_list), [id, f, root = root.get()](const auto& path) {
134 [id, f, root = root.get()](const auto& product) {
135 const auto& path = product.first;
148 auto path_list = QString::fromStdString(path).split('/');
136 auto path_list = QString::fromStdString(path).split('/');
149 auto name = *(std::cend(path_list) - 1);
137 auto name = *(std::cend(path_list) - 1);
150 auto path_item
138 auto path_item
151 = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root);
139 = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root);
152 path_item->appendChild(
140 QVariantHash metaData { { DataSourceItem::NAME_DATA_KEY, name } };
153 make_product_item({ { DataSourceItem::NAME_DATA_KEY, name } }, id));
141 std::for_each(std::cbegin(product.second), std::cend(product.second),
142 [&metaData](const auto& mdata) {
143 metaData[QString::fromStdString(mdata.first)]
144 = QString::fromStdString(mdata.second);
145 });
146 path_item->appendChild(make_product_item(metaData, id));
154 });
147 });
155 dataSourceController.setDataSourceItem(id, std::move(root));
148 dataSourceController.setDataSourceItem(id, std::move(root));
156 dataSourceController.setDataProvider(id, std::make_unique<PythonProvider>(f));
149 dataSourceController.setDataProvider(id, std::make_unique<PythonProvider>(f));
157 std::cout << "Gone there" << std::endl;
150 std::cout << "Gone there" << std::endl;
158 }
151 }
General Comments 0
You need to be logged in to leave comments. Login now